2024-11-11 02:30:43,184 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 02:30:43,197 main DEBUG Took 0.011106 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 02:30:43,198 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 02:30:43,198 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 02:30:43,199 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 02:30:43,200 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:30:43,208 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 02:30:43,223 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:30:43,225 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:30:43,226 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:30:43,226 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:30:43,227 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:30:43,228 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:30:43,229 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:30:43,230 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:30:43,230 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:30:43,231 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:30:43,232 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:30:43,233 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:30:43,234 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:30:43,234 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:30:43,235 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:30:43,235 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:30:43,236 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:30:43,236 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:30:43,237 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:30:43,237 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:30:43,238 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:30:43,238 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:30:43,239 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:30:43,239 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:30:43,240 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:30:43,240 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 02:30:43,242 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:30:43,244 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 02:30:43,247 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 02:30:43,247 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 02:30:43,249 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 02:30:43,250 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 02:30:43,259 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 02:30:43,261 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 02:30:43,263 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 02:30:43,263 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 02:30:43,264 main DEBUG createAppenders(={Console}) 2024-11-11 02:30:43,264 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-11 02:30:43,265 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 02:30:43,265 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-11 02:30:43,266 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 02:30:43,266 main DEBUG OutputStream closed 2024-11-11 02:30:43,267 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 02:30:43,267 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 02:30:43,267 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-11 02:30:43,352 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 02:30:43,354 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 02:30:43,356 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 02:30:43,357 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 02:30:43,358 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 02:30:43,359 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 02:30:43,359 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 02:30:43,360 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 02:30:43,360 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 02:30:43,361 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 02:30:43,361 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 02:30:43,362 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 02:30:43,362 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 02:30:43,363 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 02:30:43,363 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 02:30:43,364 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 02:30:43,364 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 02:30:43,365 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 02:30:43,368 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 02:30:43,368 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-11 02:30:43,368 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 02:30:43,369 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-11T02:30:43,583 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a 2024-11-11 02:30:43,586 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 02:30:43,586 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T02:30:43,594 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-11T02:30:43,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T02:30:43,634 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/cluster_15b64ed8-3a31-62e5-1ac1-ab6081429dc2, deleteOnExit=true 2024-11-11T02:30:43,635 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T02:30:43,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/test.cache.data in system properties and HBase conf 2024-11-11T02:30:43,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T02:30:43,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/hadoop.log.dir in system properties and HBase conf 2024-11-11T02:30:43,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T02:30:43,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T02:30:43,639 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T02:30:43,716 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T02:30:43,808 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T02:30:43,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T02:30:43,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T02:30:43,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T02:30:43,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T02:30:43,815 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T02:30:43,815 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T02:30:43,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T02:30:43,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T02:30:43,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T02:30:43,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/nfs.dump.dir in system properties and HBase conf 2024-11-11T02:30:43,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/java.io.tmpdir in system properties and HBase conf 2024-11-11T02:30:43,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T02:30:43,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T02:30:43,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T02:30:44,536 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T02:30:44,607 INFO [Time-limited test {}] log.Log(170): Logging initialized @2095ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T02:30:44,671 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:30:44,725 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:30:44,744 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:30:44,745 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:30:44,746 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T02:30:44,755 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:30:44,758 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:30:44,759 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:30:44,932 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/java.io.tmpdir/jetty-localhost-40011-hadoop-hdfs-3_4_1-tests_jar-_-any-4425286321897763926/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T02:30:44,938 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:40011} 2024-11-11T02:30:44,938 INFO [Time-limited test {}] server.Server(415): Started @2427ms 2024-11-11T02:30:45,263 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:30:45,273 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:30:45,274 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:30:45,274 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:30:45,275 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T02:30:45,276 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:30:45,276 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:30:45,383 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/java.io.tmpdir/jetty-localhost-44717-hadoop-hdfs-3_4_1-tests_jar-_-any-12209826735478484008/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:30:45,384 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:44717} 2024-11-11T02:30:45,385 INFO [Time-limited test {}] server.Server(415): Started @2873ms 2024-11-11T02:30:45,437 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:30:45,832 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/cluster_15b64ed8-3a31-62e5-1ac1-ab6081429dc2/data/data2/current/BP-319378453-172.17.0.2-1731292244321/current, will proceed with Du for space computation calculation, 2024-11-11T02:30:45,832 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/cluster_15b64ed8-3a31-62e5-1ac1-ab6081429dc2/data/data1/current/BP-319378453-172.17.0.2-1731292244321/current, will proceed with Du for space computation calculation, 2024-11-11T02:30:45,868 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:30:45,927 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbd923937f0808c65 with lease ID 0x81f3498be1268f5c: Processing first storage report for DS-89af9293-9bc2-4990-8f1c-6fc74b80976b from datanode DatanodeRegistration(127.0.0.1:38903, datanodeUuid=30c28baa-af47-4fc4-aa4f-725e7699da4c, infoPort=42433, infoSecurePort=0, ipcPort=46477, storageInfo=lv=-57;cid=testClusterID;nsid=2083988973;c=1731292244321) 2024-11-11T02:30:45,928 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbd923937f0808c65 with lease ID 0x81f3498be1268f5c: from storage DS-89af9293-9bc2-4990-8f1c-6fc74b80976b node DatanodeRegistration(127.0.0.1:38903, datanodeUuid=30c28baa-af47-4fc4-aa4f-725e7699da4c, infoPort=42433, infoSecurePort=0, ipcPort=46477, storageInfo=lv=-57;cid=testClusterID;nsid=2083988973;c=1731292244321), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-11T02:30:45,929 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbd923937f0808c65 with lease ID 0x81f3498be1268f5c: Processing first storage report for DS-6eadb8af-4370-4cf0-8a8c-77d3074deee5 from datanode DatanodeRegistration(127.0.0.1:38903, datanodeUuid=30c28baa-af47-4fc4-aa4f-725e7699da4c, infoPort=42433, infoSecurePort=0, ipcPort=46477, storageInfo=lv=-57;cid=testClusterID;nsid=2083988973;c=1731292244321) 2024-11-11T02:30:45,930 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbd923937f0808c65 with lease ID 0x81f3498be1268f5c: from storage DS-6eadb8af-4370-4cf0-8a8c-77d3074deee5 node DatanodeRegistration(127.0.0.1:38903, datanodeUuid=30c28baa-af47-4fc4-aa4f-725e7699da4c, infoPort=42433, infoSecurePort=0, ipcPort=46477, storageInfo=lv=-57;cid=testClusterID;nsid=2083988973;c=1731292244321), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:30:46,018 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a 2024-11-11T02:30:46,088 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/cluster_15b64ed8-3a31-62e5-1ac1-ab6081429dc2/zookeeper_0, clientPort=53153, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/cluster_15b64ed8-3a31-62e5-1ac1-ab6081429dc2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/cluster_15b64ed8-3a31-62e5-1ac1-ab6081429dc2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T02:30:46,098 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53153 2024-11-11T02:30:46,107 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:30:46,110 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:30:46,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741825_1001 (size=7) 2024-11-11T02:30:46,738 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe with version=8 2024-11-11T02:30:46,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/hbase-staging 2024-11-11T02:30:46,828 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T02:30:47,022 INFO [Time-limited test {}] client.ConnectionUtils(128): master/370bc2ade342:0 server-side Connection retries=45 2024-11-11T02:30:47,030 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:30:47,030 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T02:30:47,035 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T02:30:47,035 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:30:47,035 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T02:30:47,159 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T02:30:47,215 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T02:30:47,224 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T02:30:47,227 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T02:30:47,251 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 45236 (auto-detected) 2024-11-11T02:30:47,252 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T02:30:47,270 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42785 2024-11-11T02:30:47,289 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42785 connecting to ZooKeeper ensemble=127.0.0.1:53153 2024-11-11T02:30:47,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:427850x0, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T02:30:47,316 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42785-0x1002c9e8ca60000 connected 2024-11-11T02:30:47,342 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:30:47,345 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:30:47,353 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:30:47,356 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe, hbase.cluster.distributed=false 2024-11-11T02:30:47,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T02:30:47,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42785 2024-11-11T02:30:47,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42785 2024-11-11T02:30:47,384 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42785 2024-11-11T02:30:47,385 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42785 2024-11-11T02:30:47,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42785 2024-11-11T02:30:47,473 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/370bc2ade342:0 server-side Connection retries=45 2024-11-11T02:30:47,475 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:30:47,475 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T02:30:47,475 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T02:30:47,475 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:30:47,475 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T02:30:47,478 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T02:30:47,480 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T02:30:47,481 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38037 2024-11-11T02:30:47,483 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38037 connecting to ZooKeeper ensemble=127.0.0.1:53153 2024-11-11T02:30:47,484 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:30:47,487 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:30:47,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:380370x0, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T02:30:47,492 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:380370x0, quorum=127.0.0.1:53153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:30:47,492 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38037-0x1002c9e8ca60001 connected 2024-11-11T02:30:47,496 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T02:30:47,502 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T02:30:47,505 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T02:30:47,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T02:30:47,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38037 2024-11-11T02:30:47,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38037 2024-11-11T02:30:47,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38037 2024-11-11T02:30:47,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38037 2024-11-11T02:30:47,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38037 2024-11-11T02:30:47,530 DEBUG [M:0;370bc2ade342:42785 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;370bc2ade342:42785 2024-11-11T02:30:47,531 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/370bc2ade342,42785,1731292246873 2024-11-11T02:30:47,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:30:47,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:30:47,537 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/370bc2ade342,42785,1731292246873 2024-11-11T02:30:47,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T02:30:47,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:30:47,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:30:47,560 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T02:30:47,561 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/370bc2ade342,42785,1731292246873 from backup master directory 2024-11-11T02:30:47,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/370bc2ade342,42785,1731292246873 2024-11-11T02:30:47,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:30:47,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:30:47,564 WARN [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T02:30:47,565 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=370bc2ade342,42785,1731292246873 2024-11-11T02:30:47,566 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T02:30:47,568 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T02:30:47,622 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/hbase.id] with ID: d8a9f923-fca5-401d-96c9-535be72bc124 2024-11-11T02:30:47,622 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/.tmp/hbase.id 2024-11-11T02:30:47,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741826_1002 (size=42) 2024-11-11T02:30:48,038 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/.tmp/hbase.id]:[hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/hbase.id] 2024-11-11T02:30:48,090 INFO [master/370bc2ade342:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:30:48,095 INFO [master/370bc2ade342:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T02:30:48,113 INFO [master/370bc2ade342:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-11T02:30:48,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:30:48,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:30:48,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741827_1003 (size=196) 2024-11-11T02:30:48,552 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:30:48,555 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T02:30:48,568 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:48,572 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:30:48,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741828_1004 (size=1189) 2024-11-11T02:30:49,022 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store 2024-11-11T02:30:49,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741829_1005 (size=34) 2024-11-11T02:30:49,453 INFO [master/370bc2ade342:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T02:30:49,456 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:30:49,457 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T02:30:49,457 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:30:49,457 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:30:49,458 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T02:30:49,459 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:30:49,459 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:30:49,460 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731292249457Disabling compacts and flushes for region at 1731292249457Disabling writes for close at 1731292249458 (+1 ms)Writing region close event to WAL at 1731292249459 (+1 ms)Closed at 1731292249459 2024-11-11T02:30:49,462 WARN [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/.initializing 2024-11-11T02:30:49,462 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/WALs/370bc2ade342,42785,1731292246873 2024-11-11T02:30:49,469 INFO [master/370bc2ade342:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T02:30:49,482 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C42785%2C1731292246873, suffix=, logDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/WALs/370bc2ade342,42785,1731292246873, archiveDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/oldWALs, maxLogs=10 2024-11-11T02:30:49,502 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/WALs/370bc2ade342,42785,1731292246873/370bc2ade342%2C42785%2C1731292246873.1731292249486, exclude list is [], retry=0 2024-11-11T02:30:49,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38903,DS-89af9293-9bc2-4990-8f1c-6fc74b80976b,DISK] 2024-11-11T02:30:49,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T02:30:49,550 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/WALs/370bc2ade342,42785,1731292246873/370bc2ade342%2C42785%2C1731292246873.1731292249486 2024-11-11T02:30:49,551 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42433:42433)] 2024-11-11T02:30:49,551 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:30:49,552 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:30:49,555 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:30:49,555 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:30:49,588 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:30:49,609 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T02:30:49,612 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:49,615 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:30:49,615 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:30:49,618 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T02:30:49,618 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:49,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:30:49,620 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:30:49,622 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T02:30:49,622 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:49,623 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:30:49,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:30:49,626 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T02:30:49,626 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:49,627 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:30:49,628 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:30:49,631 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:30:49,633 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:30:49,638 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:30:49,639 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:30:49,642 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T02:30:49,646 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:30:49,651 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:30:49,653 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61727606, jitterRate=-0.08018699288368225}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T02:30:49,659 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731292249567Initializing all the Stores at 1731292249569 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731292249569Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292249570 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292249570Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292249570Cleaning up temporary data from old regions at 1731292249639 (+69 ms)Region opened successfully at 1731292249659 (+20 ms) 2024-11-11T02:30:49,660 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T02:30:49,691 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54b8555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=370bc2ade342/172.17.0.2:0 2024-11-11T02:30:49,718 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T02:30:49,728 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T02:30:49,728 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T02:30:49,730 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T02:30:49,732 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T02:30:49,736 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T02:30:49,736 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T02:30:49,760 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T02:30:49,769 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T02:30:49,770 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T02:30:49,773 INFO [master/370bc2ade342:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T02:30:49,774 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T02:30:49,776 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T02:30:49,778 INFO [master/370bc2ade342:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T02:30:49,781 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T02:30:49,782 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T02:30:49,784 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T02:30:49,785 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T02:30:49,802 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T02:30:49,803 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T02:30:49,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T02:30:49,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T02:30:49,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:30:49,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:30:49,809 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=370bc2ade342,42785,1731292246873, sessionid=0x1002c9e8ca60000, setting cluster-up flag (Was=false) 2024-11-11T02:30:49,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:30:49,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:30:49,822 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T02:30:49,823 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=370bc2ade342,42785,1731292246873 2024-11-11T02:30:49,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:30:49,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:30:49,833 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T02:30:49,835 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=370bc2ade342,42785,1731292246873 2024-11-11T02:30:49,846 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T02:30:49,911 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T02:30:49,919 INFO [master/370bc2ade342:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T02:30:49,920 INFO [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(746): ClusterId : d8a9f923-fca5-401d-96c9-535be72bc124 2024-11-11T02:30:49,923 DEBUG [RS:0;370bc2ade342:38037 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T02:30:49,926 INFO [master/370bc2ade342:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T02:30:49,927 DEBUG [RS:0;370bc2ade342:38037 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T02:30:49,927 DEBUG [RS:0;370bc2ade342:38037 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T02:30:49,929 DEBUG [RS:0;370bc2ade342:38037 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T02:30:49,930 DEBUG [RS:0;370bc2ade342:38037 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@431c69d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=370bc2ade342/172.17.0.2:0 2024-11-11T02:30:49,931 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 370bc2ade342,42785,1731292246873 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T02:30:49,938 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:30:49,938 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:30:49,938 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:30:49,939 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:30:49,939 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/370bc2ade342:0, corePoolSize=10, maxPoolSize=10 2024-11-11T02:30:49,939 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:30:49,939 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/370bc2ade342:0, corePoolSize=2, maxPoolSize=2 2024-11-11T02:30:49,939 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:30:49,940 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731292279940 2024-11-11T02:30:49,942 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T02:30:49,943 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T02:30:49,944 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;370bc2ade342:38037 2024-11-11T02:30:49,944 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:30:49,944 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T02:30:49,946 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T02:30:49,947 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T02:30:49,947 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T02:30:49,947 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T02:30:49,947 INFO [RS:0;370bc2ade342:38037 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T02:30:49,947 INFO [RS:0;370bc2ade342:38037 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T02:30:49,947 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T02:30:49,950 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:49,948 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:49,950 INFO [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(2659): reportForDuty to master=370bc2ade342,42785,1731292246873 with port=38037, startcode=1731292247443 2024-11-11T02:30:49,950 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T02:30:49,952 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T02:30:49,953 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T02:30:49,954 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T02:30:49,956 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T02:30:49,956 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T02:30:49,959 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731292249958,5,FailOnTimeoutGroup] 2024-11-11T02:30:49,960 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731292249959,5,FailOnTimeoutGroup] 2024-11-11T02:30:49,960 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:49,960 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T02:30:49,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741831_1007 (size=1321) 2024-11-11T02:30:49,961 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:49,962 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:49,963 DEBUG [RS:0;370bc2ade342:38037 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T02:30:49,996 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50085, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T02:30:50,002 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42785 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 370bc2ade342,38037,1731292247443 2024-11-11T02:30:50,003 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42785 {}] master.ServerManager(517): Registering regionserver=370bc2ade342,38037,1731292247443 2024-11-11T02:30:50,016 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe 2024-11-11T02:30:50,016 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40171 2024-11-11T02:30:50,016 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T02:30:50,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T02:30:50,020 DEBUG [RS:0;370bc2ade342:38037 {}] zookeeper.ZKUtil(111): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/370bc2ade342,38037,1731292247443 2024-11-11T02:30:50,020 WARN [RS:0;370bc2ade342:38037 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T02:30:50,020 INFO [RS:0;370bc2ade342:38037 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:30:50,021 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/WALs/370bc2ade342,38037,1731292247443 2024-11-11T02:30:50,022 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [370bc2ade342,38037,1731292247443] 2024-11-11T02:30:50,043 INFO [RS:0;370bc2ade342:38037 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T02:30:50,063 INFO [RS:0;370bc2ade342:38037 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T02:30:50,068 INFO [RS:0;370bc2ade342:38037 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T02:30:50,068 INFO [RS:0;370bc2ade342:38037 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:50,069 INFO [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T02:30:50,075 INFO [RS:0;370bc2ade342:38037 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T02:30:50,076 INFO [RS:0;370bc2ade342:38037 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:50,076 DEBUG [RS:0;370bc2ade342:38037 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:30:50,076 DEBUG [RS:0;370bc2ade342:38037 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:30:50,077 DEBUG [RS:0;370bc2ade342:38037 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:30:50,077 DEBUG [RS:0;370bc2ade342:38037 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:30:50,077 DEBUG [RS:0;370bc2ade342:38037 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:30:50,077 DEBUG [RS:0;370bc2ade342:38037 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/370bc2ade342:0, corePoolSize=2, maxPoolSize=2 2024-11-11T02:30:50,077 DEBUG [RS:0;370bc2ade342:38037 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:30:50,077 DEBUG [RS:0;370bc2ade342:38037 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:30:50,078 DEBUG [RS:0;370bc2ade342:38037 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:30:50,078 DEBUG [RS:0;370bc2ade342:38037 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:30:50,078 DEBUG [RS:0;370bc2ade342:38037 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:30:50,078 DEBUG [RS:0;370bc2ade342:38037 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:30:50,078 DEBUG [RS:0;370bc2ade342:38037 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/370bc2ade342:0, corePoolSize=3, maxPoolSize=3 2024-11-11T02:30:50,079 DEBUG [RS:0;370bc2ade342:38037 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0, corePoolSize=3, maxPoolSize=3 2024-11-11T02:30:50,079 INFO [RS:0;370bc2ade342:38037 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:50,080 INFO [RS:0;370bc2ade342:38037 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:50,080 INFO [RS:0;370bc2ade342:38037 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:50,080 INFO [RS:0;370bc2ade342:38037 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:50,080 INFO [RS:0;370bc2ade342:38037 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:50,080 INFO [RS:0;370bc2ade342:38037 {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,38037,1731292247443-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T02:30:50,098 INFO [RS:0;370bc2ade342:38037 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T02:30:50,100 INFO [RS:0;370bc2ade342:38037 {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,38037,1731292247443-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:50,100 INFO [RS:0;370bc2ade342:38037 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:50,100 INFO [RS:0;370bc2ade342:38037 {}] regionserver.Replication(171): 370bc2ade342,38037,1731292247443 started 2024-11-11T02:30:50,117 INFO [RS:0;370bc2ade342:38037 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:50,117 INFO [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1482): Serving as 370bc2ade342,38037,1731292247443, RpcServer on 370bc2ade342/172.17.0.2:38037, sessionid=0x1002c9e8ca60001 2024-11-11T02:30:50,118 DEBUG [RS:0;370bc2ade342:38037 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T02:30:50,118 DEBUG [RS:0;370bc2ade342:38037 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 370bc2ade342,38037,1731292247443 2024-11-11T02:30:50,118 DEBUG [RS:0;370bc2ade342:38037 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '370bc2ade342,38037,1731292247443' 2024-11-11T02:30:50,118 DEBUG [RS:0;370bc2ade342:38037 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T02:30:50,119 DEBUG [RS:0;370bc2ade342:38037 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T02:30:50,120 DEBUG [RS:0;370bc2ade342:38037 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T02:30:50,120 DEBUG [RS:0;370bc2ade342:38037 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T02:30:50,120 DEBUG [RS:0;370bc2ade342:38037 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 370bc2ade342,38037,1731292247443 2024-11-11T02:30:50,120 DEBUG [RS:0;370bc2ade342:38037 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '370bc2ade342,38037,1731292247443' 2024-11-11T02:30:50,120 DEBUG [RS:0;370bc2ade342:38037 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T02:30:50,121 DEBUG [RS:0;370bc2ade342:38037 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T02:30:50,122 DEBUG [RS:0;370bc2ade342:38037 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T02:30:50,122 INFO [RS:0;370bc2ade342:38037 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T02:30:50,122 INFO [RS:0;370bc2ade342:38037 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T02:30:50,233 INFO [RS:0;370bc2ade342:38037 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T02:30:50,236 INFO [RS:0;370bc2ade342:38037 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C38037%2C1731292247443, suffix=, logDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/WALs/370bc2ade342,38037,1731292247443, archiveDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/oldWALs, maxLogs=32 2024-11-11T02:30:50,250 DEBUG [RS:0;370bc2ade342:38037 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/WALs/370bc2ade342,38037,1731292247443/370bc2ade342%2C38037%2C1731292247443.1731292250238, exclude list is [], retry=0 2024-11-11T02:30:50,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38903,DS-89af9293-9bc2-4990-8f1c-6fc74b80976b,DISK] 2024-11-11T02:30:50,257 INFO [RS:0;370bc2ade342:38037 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/WALs/370bc2ade342,38037,1731292247443/370bc2ade342%2C38037%2C1731292247443.1731292250238 2024-11-11T02:30:50,257 DEBUG [RS:0;370bc2ade342:38037 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42433:42433)] 2024-11-11T02:30:50,364 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T02:30:50,365 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe 2024-11-11T02:30:50,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741833_1009 (size=32) 2024-11-11T02:30:50,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:30:50,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T02:30:50,794 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T02:30:50,794 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:50,795 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:30:50,796 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T02:30:50,799 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T02:30:50,799 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:50,800 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:30:50,800 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T02:30:50,803 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T02:30:50,803 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:50,804 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:30:50,804 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T02:30:50,806 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T02:30:50,806 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:50,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:30:50,807 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T02:30:50,809 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740 2024-11-11T02:30:50,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740 2024-11-11T02:30:50,813 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T02:30:50,813 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T02:30:50,814 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:30:50,816 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T02:30:50,820 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:30:50,821 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66711034, jitterRate=-0.005928128957748413}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:30:50,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731292250786Initializing all the Stores at 1731292250790 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731292250790Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731292250790Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292250790Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731292250790Cleaning up temporary data from old regions at 1731292250813 (+23 ms)Region opened successfully at 1731292250824 (+11 ms) 2024-11-11T02:30:50,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T02:30:50,824 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T02:30:50,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T02:30:50,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T02:30:50,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T02:30:50,826 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T02:30:50,826 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731292250824Disabling compacts and flushes for region at 1731292250824Disabling writes for close at 1731292250824Writing region close event to WAL at 1731292250825 (+1 ms)Closed at 1731292250825 2024-11-11T02:30:50,829 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:30:50,829 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T02:30:50,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T02:30:50,841 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T02:30:50,843 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T02:30:51,002 DEBUG [370bc2ade342:42785 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T02:30:51,014 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:30:51,019 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 370bc2ade342,38037,1731292247443, state=OPENING 2024-11-11T02:30:51,024 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T02:30:51,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:30:51,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:30:51,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:30:51,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:30:51,028 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T02:30:51,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:30:51,211 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T02:30:51,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40301, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T02:30:51,224 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T02:30:51,224 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:30:51,225 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T02:30:51,228 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C38037%2C1731292247443.meta, suffix=.meta, logDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/WALs/370bc2ade342,38037,1731292247443, archiveDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/oldWALs, maxLogs=32 2024-11-11T02:30:51,247 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/WALs/370bc2ade342,38037,1731292247443/370bc2ade342%2C38037%2C1731292247443.meta.1731292251231.meta, exclude list is [], retry=0 2024-11-11T02:30:51,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38903,DS-89af9293-9bc2-4990-8f1c-6fc74b80976b,DISK] 2024-11-11T02:30:51,254 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/WALs/370bc2ade342,38037,1731292247443/370bc2ade342%2C38037%2C1731292247443.meta.1731292251231.meta 2024-11-11T02:30:51,254 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42433:42433)] 2024-11-11T02:30:51,254 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:30:51,256 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T02:30:51,258 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T02:30:51,262 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T02:30:51,266 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T02:30:51,267 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:30:51,267 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T02:30:51,267 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T02:30:51,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T02:30:51,272 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T02:30:51,272 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:51,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:30:51,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T02:30:51,275 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T02:30:51,275 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:51,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:30:51,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T02:30:51,277 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T02:30:51,277 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:51,278 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:30:51,278 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T02:30:51,280 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T02:30:51,280 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:51,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:30:51,281 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T02:30:51,282 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740 2024-11-11T02:30:51,285 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740 2024-11-11T02:30:51,287 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T02:30:51,287 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T02:30:51,288 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:30:51,291 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T02:30:51,292 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71377851, jitterRate=0.06361286342144012}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:30:51,293 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T02:30:51,294 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731292251268Writing region info on filesystem at 1731292251268Initializing all the Stores at 1731292251270 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731292251270Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731292251270Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292251270Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731292251270Cleaning up temporary data from old regions at 1731292251287 (+17 ms)Running coprocessor post-open hooks at 1731292251293 (+6 ms)Region opened successfully at 1731292251294 (+1 ms) 2024-11-11T02:30:51,300 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731292251203 2024-11-11T02:30:51,310 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T02:30:51,310 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T02:30:51,312 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:30:51,314 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 370bc2ade342,38037,1731292247443, state=OPEN 2024-11-11T02:30:51,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T02:30:51,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T02:30:51,317 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:30:51,318 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:30:51,318 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:51,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T02:30:51,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=370bc2ade342,38037,1731292247443 in 288 msec 2024-11-11T02:30:51,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T02:30:51,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 493 msec 2024-11-11T02:30:51,334 DEBUG [PEWorker-4 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:30:51,334 INFO [PEWorker-4 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T02:30:51,351 DEBUG [PEWorker-4 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:30:51,352 DEBUG [PEWorker-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:30:51,368 DEBUG [PEWorker-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:30:51,371 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32865, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:30:51,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5150 sec 2024-11-11T02:30:51,390 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731292251390, completionTime=-1 2024-11-11T02:30:51,392 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T02:30:51,393 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T02:30:51,415 INFO [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-11T02:30:51,415 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731292311415 2024-11-11T02:30:51,415 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731292371415 2024-11-11T02:30:51,415 INFO [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-11-11T02:30:51,417 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,42785,1731292246873-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:51,418 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,42785,1731292246873-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:51,418 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,42785,1731292246873-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:51,419 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-370bc2ade342:42785, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:51,419 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:51,420 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:51,425 DEBUG [master/370bc2ade342:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T02:30:51,447 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.882sec 2024-11-11T02:30:51,448 INFO [master/370bc2ade342:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T02:30:51,449 INFO [master/370bc2ade342:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T02:30:51,450 INFO [master/370bc2ade342:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T02:30:51,451 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T02:30:51,451 INFO [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T02:30:51,452 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,42785,1731292246873-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T02:30:51,452 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,42785,1731292246873-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T02:30:51,460 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T02:30:51,461 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T02:30:51,461 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,42785,1731292246873-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:30:51,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:51,536 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T02:30:51,536 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T02:30:51,539 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:30:51,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:30:51,549 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:30:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:30:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:30:51,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775efbc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:51,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:30:51,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:30:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:30:51,559 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:30:51,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:51,562 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:30:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:30:51,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:30:51,572 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:30:51,592 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=370bc2ade342,42785,1731292246873 2024-11-11T02:30:51,603 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=1135, ProcessCount=11, AvailableMemoryMB=5797 2024-11-11T02:30:51,625 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T02:30:51,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 370bc2ade342,42785,1731292246873 2024-11-11T02:30:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c35b008 2024-11-11T02:30:51,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T02:30:51,637 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T02:30:51,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T02:30:51,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:30:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T02:30:51,656 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T02:30:51,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-11T02:30:51,658 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:51,661 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T02:30:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T02:30:51,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741835_1011 (size=1061) 2024-11-11T02:30:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T02:30:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T02:30:52,078 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe 2024-11-11T02:30:52,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741836_1012 (size=53) 2024-11-11T02:30:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T02:30:52,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:30:52,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 21635db8226dc83255808b1cf0db52eb, disabling compactions & flushes 2024-11-11T02:30:52,497 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:52,497 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:52,497 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. after waiting 0 ms 2024-11-11T02:30:52,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:52,498 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:52,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 21635db8226dc83255808b1cf0db52eb: Waiting for close lock at 1731292252496Disabling compacts and flushes for region at 1731292252496Disabling writes for close at 1731292252497 (+1 ms)Writing region close event to WAL at 1731292252498 (+1 ms)Closed at 1731292252498 2024-11-11T02:30:52,501 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T02:30:52,505 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731292252501"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731292252501"}]},"ts":"1731292252501"} 2024-11-11T02:30:52,510 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T02:30:52,511 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T02:30:52,514 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292252512"}]},"ts":"1731292252512"} 2024-11-11T02:30:52,518 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T02:30:52,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21635db8226dc83255808b1cf0db52eb, ASSIGN}] 2024-11-11T02:30:52,522 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21635db8226dc83255808b1cf0db52eb, ASSIGN 2024-11-11T02:30:52,524 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21635db8226dc83255808b1cf0db52eb, ASSIGN; state=OFFLINE, location=370bc2ade342,38037,1731292247443; forceNewPlan=false, retain=false 2024-11-11T02:30:52,678 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=21635db8226dc83255808b1cf0db52eb, regionState=OPENING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:30:52,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21635db8226dc83255808b1cf0db52eb, ASSIGN because future has completed 2024-11-11T02:30:52,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:30:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T02:30:52,851 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:52,851 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:30:52,852 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:52,852 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:30:52,852 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:52,852 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:52,856 INFO [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:52,859 INFO [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:30:52,859 INFO [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21635db8226dc83255808b1cf0db52eb columnFamilyName A 2024-11-11T02:30:52,859 DEBUG [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:52,861 INFO [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] regionserver.HStore(327): Store=21635db8226dc83255808b1cf0db52eb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:30:52,861 INFO [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:52,863 INFO [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:30:52,863 INFO [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21635db8226dc83255808b1cf0db52eb columnFamilyName B 2024-11-11T02:30:52,863 DEBUG [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:52,864 INFO [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] regionserver.HStore(327): Store=21635db8226dc83255808b1cf0db52eb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:30:52,864 INFO [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:52,866 INFO [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:30:52,866 INFO [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21635db8226dc83255808b1cf0db52eb columnFamilyName C 2024-11-11T02:30:52,866 DEBUG [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:30:52,867 INFO [StoreOpener-21635db8226dc83255808b1cf0db52eb-1 {}] regionserver.HStore(327): Store=21635db8226dc83255808b1cf0db52eb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:30:52,868 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:52,868 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:52,869 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:52,870 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:52,872 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:52,872 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:52,873 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:30:52,875 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:52,878 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:30:52,879 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 21635db8226dc83255808b1cf0db52eb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73841691, jitterRate=0.10032694041728973}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:30:52,879 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:52,880 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 21635db8226dc83255808b1cf0db52eb: Running coprocessor pre-open hook at 1731292252853Writing region info on filesystem at 1731292252853Initializing all the Stores at 1731292252855 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292252855Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292252855Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292252855Cleaning up temporary data from old regions at 1731292252872 (+17 ms)Running coprocessor post-open hooks at 1731292252879 (+7 ms)Region opened successfully at 1731292252880 (+1 ms) 2024-11-11T02:30:52,882 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., pid=6, masterSystemTime=1731292252844 2024-11-11T02:30:52,885 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:52,886 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:52,887 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=21635db8226dc83255808b1cf0db52eb, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:30:52,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:30:52,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T02:30:52,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 in 203 msec 2024-11-11T02:30:52,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T02:30:52,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21635db8226dc83255808b1cf0db52eb, ASSIGN in 376 msec 2024-11-11T02:30:52,902 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T02:30:52,903 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292252902"}]},"ts":"1731292252902"} 2024-11-11T02:30:52,910 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T02:30:52,912 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T02:30:52,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2670 sec 2024-11-11T02:30:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T02:30:53,820 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:30:53,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:30:53,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:30:53,831 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:30:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:30:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:30:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d6336a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:30:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:30:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:30:53,834 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:30:53,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d7323ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,836 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:30:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:30:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:30:53,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:30:53,843 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T02:30:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 370bc2ade342,42785,1731292246873 2024-11-11T02:30:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@58b5dcfd 2024-11-11T02:30:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T02:30:53,849 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T02:30:53,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,853 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:30:53,854 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:30:53,855 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:30:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:30:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:30:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703ed473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:30:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:30:53,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:30:53,858 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:30:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3937ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:30:53,861 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:30:53,862 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:30:53,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:30:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:30:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffd0298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:30:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:30:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:30:53,865 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:30:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bed14c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:30:53,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:30:53,868 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:30:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:30:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:30:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af2c0a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:30:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:30:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:30:53,871 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:30:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1612488f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:30:53,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:30:53,875 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:30:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:30:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:30:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9dd8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:30:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:30:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:30:53,878 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:30:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3ac293, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:30:53,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:30:53,882 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:30:53,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:30:53,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:30:53,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4331d0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:30:53,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:30:53,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:30:53,885 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:30:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17daddbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:30:53,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:30:53,889 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:30:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:30:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:30:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60486904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:30:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:30:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:30:53,893 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:30:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f58bfae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:30:53,895 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:30:53,896 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:30:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:30:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:30:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24aa1c15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:30:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:30:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:30:53,898 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:30:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fce0532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,901 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:30:53,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:30:53,902 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:30:53,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:30:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:30:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee22a4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:30:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:30:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:30:53,904 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:30:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b92dc16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:30:53,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:30:53,907 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:30:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:30:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:30:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83f8e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:30:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:30:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:30:53,910 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:30:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c944594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:30:53,915 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:30:53,915 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:30:53,915 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:30:53,915 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:30:53,915 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:30:53,916 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:30:53,916 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:30:53,917 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:30:53,916 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:30:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:30:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:30:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:30:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:30:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:30:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:30:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:30:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:30:53,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:30:53,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:30:53,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:30:53,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:30:53,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:30:53,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:30:53,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:30:53,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:30:53,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:30:53,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:30:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:30:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:30:53,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:30:53,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:30:53,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:30:53,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:30:53,920 DEBUG [Thread-156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:30:53,925 DEBUG [Thread-156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:30:53,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:30:53,927 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:30:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:30:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:30:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:30:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:30:53,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:30:53,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:30:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:30:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-11T02:30:53,941 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:30:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:30:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:30:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T02:30:53,944 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:30:53,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:30:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:54,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:30:54,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:30:54,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:30:54,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:30:54,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:30:54,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:30:54,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:30:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T02:30:54,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/a71dfdc747014683b045c5a77b0389dc is 50, key is test_row_0/A:col10/1731292253998/Put/seqid=0 2024-11-11T02:30:54,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T02:30:54,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:54,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:54,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:54,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741837_1013 (size=12001) 2024-11-11T02:30:54,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/a71dfdc747014683b045c5a77b0389dc 2024-11-11T02:30:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/c9f0157135014bb2a526cfb94967a35c is 50, key is test_row_0/B:col10/1731292253998/Put/seqid=0 2024-11-11T02:30:54,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292314203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292314174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292314204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292314213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292314217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741838_1014 (size=12001) 2024-11-11T02:30:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/c9f0157135014bb2a526cfb94967a35c 2024-11-11T02:30:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T02:30:54,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T02:30:54,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/f34882f4d61745e19ac70cac86cf2955 is 50, key is test_row_0/C:col10/1731292253998/Put/seqid=0 2024-11-11T02:30:54,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:54,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:54,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:54,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741839_1015 (size=12001) 2024-11-11T02:30:54,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292314356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292314357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292314369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292314369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292314372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T02:30:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:54,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T02:30:54,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292314577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292314578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292314587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292314587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292314587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T02:30:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:54,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/f34882f4d61745e19ac70cac86cf2955 2024-11-11T02:30:54,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/a71dfdc747014683b045c5a77b0389dc as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/a71dfdc747014683b045c5a77b0389dc 2024-11-11T02:30:54,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/a71dfdc747014683b045c5a77b0389dc, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T02:30:54,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/c9f0157135014bb2a526cfb94967a35c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/c9f0157135014bb2a526cfb94967a35c 2024-11-11T02:30:54,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T02:30:54,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:54,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:54,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:54,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/c9f0157135014bb2a526cfb94967a35c, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T02:30:54,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/f34882f4d61745e19ac70cac86cf2955 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/f34882f4d61745e19ac70cac86cf2955 2024-11-11T02:30:54,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/f34882f4d61745e19ac70cac86cf2955, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T02:30:54,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 21635db8226dc83255808b1cf0db52eb in 806ms, sequenceid=14, compaction requested=false 2024-11-11T02:30:54,828 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T02:30:54,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:30:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:54,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=114.05 KB heapSize=339.89 KB 2024-11-11T02:30:54,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:30:54,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:30:54,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:30:54,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:30:54,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:30:54,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:30:54,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T02:30:54,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:54,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:54,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:54,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:54,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/e5ca1ea7fc95406cb50f7e7b0bc5ba6b is 50, key is test_row_0/A:col10/1731292254199/Put/seqid=0 2024-11-11T02:30:54,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741840_1016 (size=16681) 2024-11-11T02:30:54,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/e5ca1ea7fc95406cb50f7e7b0bc5ba6b 2024-11-11T02:30:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292314953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292314954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292314957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292314967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:54,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292314973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/a839992aca6845fdbc061dc0da01ddcf is 50, key is test_row_0/B:col10/1731292254199/Put/seqid=0 2024-11-11T02:30:55,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741841_1017 (size=12001) 2024-11-11T02:30:55,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/a839992aca6845fdbc061dc0da01ddcf 2024-11-11T02:30:55,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/ec8c97650b8948fb9666cf337df8f321 is 50, key is test_row_0/C:col10/1731292254199/Put/seqid=0 2024-11-11T02:30:55,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741842_1018 (size=12001) 2024-11-11T02:30:55,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/ec8c97650b8948fb9666cf337df8f321 2024-11-11T02:30:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T02:30:55,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T02:30:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:55,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:55,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/e5ca1ea7fc95406cb50f7e7b0bc5ba6b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/e5ca1ea7fc95406cb50f7e7b0bc5ba6b 2024-11-11T02:30:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:55,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292315086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292315087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292315087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292315087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292315098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/e5ca1ea7fc95406cb50f7e7b0bc5ba6b, entries=250, sequenceid=35, filesize=16.3 K 2024-11-11T02:30:55,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/a839992aca6845fdbc061dc0da01ddcf as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/a839992aca6845fdbc061dc0da01ddcf 2024-11-11T02:30:55,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/a839992aca6845fdbc061dc0da01ddcf, entries=150, sequenceid=35, filesize=11.7 K 2024-11-11T02:30:55,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/ec8c97650b8948fb9666cf337df8f321 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/ec8c97650b8948fb9666cf337df8f321 2024-11-11T02:30:55,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/ec8c97650b8948fb9666cf337df8f321, entries=150, sequenceid=35, filesize=11.7 K 2024-11-11T02:30:55,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 21635db8226dc83255808b1cf0db52eb in 269ms, sequenceid=35, compaction requested=false 2024-11-11T02:30:55,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:30:55,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T02:30:55,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:55,247 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T02:30:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:30:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:30:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:30:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:30:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:30:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:30:55,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/7460008f54044217ab382ce99670cc38 is 50, key is test_row_0/A:col10/1731292254963/Put/seqid=0 2024-11-11T02:30:55,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741843_1019 (size=12001) 2024-11-11T02:30:55,283 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/7460008f54044217ab382ce99670cc38 2024-11-11T02:30:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:55,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:55,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/5f42d83d709a491c99615d327b87e9c3 is 50, key is test_row_0/B:col10/1731292254963/Put/seqid=0 2024-11-11T02:30:55,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741844_1020 (size=12001) 2024-11-11T02:30:55,338 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/5f42d83d709a491c99615d327b87e9c3 2024-11-11T02:30:55,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/188b7bff934e4f1c80f647a436612340 is 50, key is test_row_0/C:col10/1731292254963/Put/seqid=0 2024-11-11T02:30:55,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731292255346, max timestamp=1731292255364Num uniques 859; 2024-11-11T02:30:55,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741845_1021 (size=12001) 2024-11-11T02:30:55,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:30:55,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292255346, max timestamp=1731292255364Num uniques 859; 2024-11-11T02:30:55,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:30:55,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292255346, max timestamp=1731292255364Num uniques 859; 2024-11-11T02:30:55,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:30:55,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292315413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292315416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292315421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292315431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292315418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292315535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292315546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292315546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292315547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292315547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292315746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292315767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292315767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292315767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292315767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:55,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:55,775 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/188b7bff934e4f1c80f647a436612340 2024-11-11T02:30:55,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/7460008f54044217ab382ce99670cc38 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/7460008f54044217ab382ce99670cc38 2024-11-11T02:30:55,804 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/7460008f54044217ab382ce99670cc38, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T02:30:55,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/5f42d83d709a491c99615d327b87e9c3 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5f42d83d709a491c99615d327b87e9c3 2024-11-11T02:30:55,825 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5f42d83d709a491c99615d327b87e9c3, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T02:30:55,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/188b7bff934e4f1c80f647a436612340 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/188b7bff934e4f1c80f647a436612340 2024-11-11T02:30:55,841 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/188b7bff934e4f1c80f647a436612340, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T02:30:55,844 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 21635db8226dc83255808b1cf0db52eb in 596ms, sequenceid=46, compaction requested=true 2024-11-11T02:30:55,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:30:55,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:55,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-11T02:30:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-11T02:30:55,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T02:30:55,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9020 sec 2024-11-11T02:30:55,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9250 sec 2024-11-11T02:30:56,038 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T02:30:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:56,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T02:30:56,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:30:56,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:30:56,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:30:56,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:30:56,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:30:56,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:30:56,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/97c9cb2a95164738b4643ff6fb493552 is 50, key is test_row_0/A:col10/1731292255427/Put/seqid=0 2024-11-11T02:30:56,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292316085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292316085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292316085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T02:30:56,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292316089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,097 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:30:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:30:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292316092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-11T02:30:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T02:30:56,102 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:30:56,104 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:30:56,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:30:56,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741846_1022 (size=14341) 2024-11-11T02:30:56,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292316196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T02:30:56,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292316206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292316207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T02:30:56,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:56,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:56,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:56,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T02:30:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:56,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:56,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T02:30:56,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292316416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292316426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292316426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/97c9cb2a95164738b4643ff6fb493552 2024-11-11T02:30:56,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/526392d0053740698bbd25ec19b213e0 is 50, key is test_row_0/B:col10/1731292255427/Put/seqid=0 2024-11-11T02:30:56,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741847_1023 (size=12001) 2024-11-11T02:30:56,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T02:30:56,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:56,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:56,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:56,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292316596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292316596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T02:30:56,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T02:30:56,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:56,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:56,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:56,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292316726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292316735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292316746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:56,781 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T02:30:56,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T02:30:56,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:56,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:56,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:56,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:56,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/526392d0053740698bbd25ec19b213e0 2024-11-11T02:30:56,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/5dd716b75be046d6ab67bdc435b7d083 is 50, key is test_row_0/C:col10/1731292255427/Put/seqid=0 2024-11-11T02:30:56,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741848_1024 (size=12001) 2024-11-11T02:30:57,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T02:30:57,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:57,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:57,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:57,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:57,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:57,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T02:30:57,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:57,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:57,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:57,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:57,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:57,210 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T02:30:57,211 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T02:30:57,214 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:30:57,214 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T02:30:57,214 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T02:30:57,215 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T02:30:57,216 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T02:30:57,216 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T02:30:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T02:30:57,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292317246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:57,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:57,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:57,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:57,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292317256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:57,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292317266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:57,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T02:30:57,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:57,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:57,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:57,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:57,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:57,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/5dd716b75be046d6ab67bdc435b7d083 2024-11-11T02:30:57,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/97c9cb2a95164738b4643ff6fb493552 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/97c9cb2a95164738b4643ff6fb493552 2024-11-11T02:30:57,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/97c9cb2a95164738b4643ff6fb493552, entries=200, sequenceid=82, filesize=14.0 K 2024-11-11T02:30:57,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/526392d0053740698bbd25ec19b213e0 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/526392d0053740698bbd25ec19b213e0 2024-11-11T02:30:57,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/526392d0053740698bbd25ec19b213e0, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T02:30:57,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/5dd716b75be046d6ab67bdc435b7d083 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/5dd716b75be046d6ab67bdc435b7d083 2024-11-11T02:30:57,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/5dd716b75be046d6ab67bdc435b7d083, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T02:30:57,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 21635db8226dc83255808b1cf0db52eb in 1385ms, sequenceid=82, compaction requested=true 2024-11-11T02:30:57,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:30:57,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:30:57,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:30:57,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:30:57,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:30:57,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:30:57,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:30:57,465 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:30:57,467 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:30:57,471 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:30:57,473 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/B is initiating minor compaction (all files) 2024-11-11T02:30:57,474 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/B in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:57,474 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/c9f0157135014bb2a526cfb94967a35c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/a839992aca6845fdbc061dc0da01ddcf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5f42d83d709a491c99615d327b87e9c3, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/526392d0053740698bbd25ec19b213e0] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=46.9 K 2024-11-11T02:30:57,476 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting c9f0157135014bb2a526cfb94967a35c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731292253939 2024-11-11T02:30:57,478 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:30:57,478 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/A is initiating minor compaction (all files) 2024-11-11T02:30:57,478 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/A in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:57,478 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/a71dfdc747014683b045c5a77b0389dc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/e5ca1ea7fc95406cb50f7e7b0bc5ba6b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/7460008f54044217ab382ce99670cc38, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/97c9cb2a95164738b4643ff6fb493552] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=53.7 K 2024-11-11T02:30:57,480 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting a839992aca6845fdbc061dc0da01ddcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731292254188 2024-11-11T02:30:57,480 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting a71dfdc747014683b045c5a77b0389dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731292253939 2024-11-11T02:30:57,481 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f42d83d709a491c99615d327b87e9c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731292254928 2024-11-11T02:30:57,482 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 526392d0053740698bbd25ec19b213e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731292255380 2024-11-11T02:30:57,483 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5ca1ea7fc95406cb50f7e7b0bc5ba6b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731292254188 2024-11-11T02:30:57,485 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7460008f54044217ab382ce99670cc38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731292254928 2024-11-11T02:30:57,487 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97c9cb2a95164738b4643ff6fb493552, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731292255380 2024-11-11T02:30:57,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T02:30:57,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:57,500 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:30:57,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:30:57,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:30:57,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:30:57,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:30:57,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:30:57,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:30:57,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/6718abb40e5e4fe9b98e4f7d44e2b1b4 is 50, key is test_row_0/A:col10/1731292256077/Put/seqid=0 2024-11-11T02:30:57,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741849_1025 (size=12001) 2024-11-11T02:30:57,549 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#B#compaction#13 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:30:57,550 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/d28e927ddd7c47648f5d6e777edf1d80 is 50, key is test_row_0/B:col10/1731292255427/Put/seqid=0 2024-11-11T02:30:57,550 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#A#compaction#14 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:30:57,551 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/3c0aa4b48f5642e3a92507054a676284 is 50, key is test_row_0/A:col10/1731292255427/Put/seqid=0 2024-11-11T02:30:57,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741850_1026 (size=12139) 2024-11-11T02:30:57,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741851_1027 (size=12139) 2024-11-11T02:30:57,581 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/3c0aa4b48f5642e3a92507054a676284 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/3c0aa4b48f5642e3a92507054a676284 2024-11-11T02:30:57,608 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/A of 21635db8226dc83255808b1cf0db52eb into 3c0aa4b48f5642e3a92507054a676284(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:30:57,608 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:30:57,611 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/A, priority=12, startTime=1731292257446; duration=0sec 2024-11-11T02:30:57,612 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:30:57,612 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:A 2024-11-11T02:30:57,612 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:30:57,615 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:30:57,615 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/C is initiating minor compaction (all files) 2024-11-11T02:30:57,616 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/C in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:57,616 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/f34882f4d61745e19ac70cac86cf2955, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/ec8c97650b8948fb9666cf337df8f321, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/188b7bff934e4f1c80f647a436612340, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/5dd716b75be046d6ab67bdc435b7d083] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=46.9 K 2024-11-11T02:30:57,617 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting f34882f4d61745e19ac70cac86cf2955, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731292253939 2024-11-11T02:30:57,618 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec8c97650b8948fb9666cf337df8f321, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731292254188 2024-11-11T02:30:57,619 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 188b7bff934e4f1c80f647a436612340, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731292254928 2024-11-11T02:30:57,621 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dd716b75be046d6ab67bdc435b7d083, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731292255380 2024-11-11T02:30:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:57,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:57,651 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#C#compaction#15 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:30:57,652 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/1d7ce294ffe845329b590b22b96fe8b1 is 50, key is test_row_0/C:col10/1731292255427/Put/seqid=0 2024-11-11T02:30:57,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741852_1028 (size=12139) 2024-11-11T02:30:57,689 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/1d7ce294ffe845329b590b22b96fe8b1 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1d7ce294ffe845329b590b22b96fe8b1 2024-11-11T02:30:57,707 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/C of 21635db8226dc83255808b1cf0db52eb into 1d7ce294ffe845329b590b22b96fe8b1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:30:57,707 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:30:57,707 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/C, priority=12, startTime=1731292257465; duration=0sec 2024-11-11T02:30:57,708 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:30:57,708 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:C 2024-11-11T02:30:57,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292257705, max timestamp=1731292257746Num uniques 859; 2024-11-11T02:30:57,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:30:57,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292257705, max timestamp=1731292257746Num uniques 859; 2024-11-11T02:30:57,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:30:57,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292257705, max timestamp=1731292257746Num uniques 859; 2024-11-11T02:30:57,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:30:57,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292317830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:57,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292317833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:57,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:57,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:57,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:57,949 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/6718abb40e5e4fe9b98e4f7d44e2b1b4 2024-11-11T02:30:57,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292317945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:57,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:57,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292317946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:57,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/e58149d7bf434c80bae4e8a5f40c1149 is 50, key is test_row_0/B:col10/1731292256077/Put/seqid=0 2024-11-11T02:30:57,983 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/d28e927ddd7c47648f5d6e777edf1d80 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/d28e927ddd7c47648f5d6e777edf1d80 2024-11-11T02:30:57,995 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/B of 21635db8226dc83255808b1cf0db52eb into d28e927ddd7c47648f5d6e777edf1d80(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:30:57,995 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:30:57,995 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/B, priority=12, startTime=1731292257460; duration=0sec 2024-11-11T02:30:57,996 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:30:57,996 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:B 2024-11-11T02:30:58,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741853_1029 (size=12001) 2024-11-11T02:30:58,004 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/e58149d7bf434c80bae4e8a5f40c1149 2024-11-11T02:30:58,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/743117e6d6ae4ef099e790005af0c8a6 is 50, key is test_row_0/C:col10/1731292256077/Put/seqid=0 2024-11-11T02:30:58,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741854_1030 (size=12001) 2024-11-11T02:30:58,039 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/743117e6d6ae4ef099e790005af0c8a6 2024-11-11T02:30:58,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/6718abb40e5e4fe9b98e4f7d44e2b1b4 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/6718abb40e5e4fe9b98e4f7d44e2b1b4 2024-11-11T02:30:58,062 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/6718abb40e5e4fe9b98e4f7d44e2b1b4, entries=150, sequenceid=92, filesize=11.7 K 2024-11-11T02:30:58,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/e58149d7bf434c80bae4e8a5f40c1149 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e58149d7bf434c80bae4e8a5f40c1149 2024-11-11T02:30:58,074 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e58149d7bf434c80bae4e8a5f40c1149, entries=150, sequenceid=92, filesize=11.7 K 2024-11-11T02:30:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/743117e6d6ae4ef099e790005af0c8a6 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/743117e6d6ae4ef099e790005af0c8a6 2024-11-11T02:30:58,091 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/743117e6d6ae4ef099e790005af0c8a6, entries=150, sequenceid=92, filesize=11.7 K 2024-11-11T02:30:58,094 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 21635db8226dc83255808b1cf0db52eb in 593ms, sequenceid=92, compaction requested=false 2024-11-11T02:30:58,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:30:58,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:58,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T02:30:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-11T02:30:58,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T02:30:58,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9920 sec 2024-11-11T02:30:58,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.0010 sec 2024-11-11T02:30:58,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-11T02:30:58,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:30:58,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:30:58,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:30:58,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:30:58,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:30:58,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:30:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:30:58,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/131b6b595f514be693e07ba977287f00 is 50, key is test_row_0/A:col10/1731292257821/Put/seqid=0 2024-11-11T02:30:58,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292318206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:58,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:58,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292318208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:58,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741855_1031 (size=12101) 2024-11-11T02:30:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T02:30:58,246 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:30:58,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:30:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-11T02:30:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T02:30:58,253 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:30:58,254 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:30:58,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:30:58,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292318268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:58,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292318268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:58,276 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in 21635db8226dc83255808b1cf0db52eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:30:58,276 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in 21635db8226dc83255808b1cf0db52eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:30:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:58,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292318275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:58,280 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in 21635db8226dc83255808b1cf0db52eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:30:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:58,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292318326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292318326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T02:30:58,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T02:30:58,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:58,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:58,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:58,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:58,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:58,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292318536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:58,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:58,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292318536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:58,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:58,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T02:30:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:58,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:58,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:58,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T02:30:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:58,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/131b6b595f514be693e07ba977287f00 2024-11-11T02:30:58,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/929dcd21333347c6838a888735ab9247 is 50, key is test_row_0/B:col10/1731292257821/Put/seqid=0 2024-11-11T02:30:58,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741856_1032 (size=12101) 2024-11-11T02:30:58,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T02:30:58,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:58,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:58,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:58,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:58,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:58,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292318846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292318846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:58,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:58,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:58,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T02:30:58,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:58,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:58,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:58,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:58,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T02:30:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:59,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T02:30:59,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:59,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:59,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:59,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:59,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:59,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:59,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/929dcd21333347c6838a888735ab9247 2024-11-11T02:30:59,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/a17859e4629343cd974429f3ca2b2a31 is 50, key is test_row_0/C:col10/1731292257821/Put/seqid=0 2024-11-11T02:30:59,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741857_1033 (size=12101) 2024-11-11T02:30:59,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T02:30:59,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:59,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:59,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:59,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:59,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:59,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:59,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T02:30:59,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:59,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:59,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:59,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:59,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:59,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292319356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:59,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:30:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292319365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:30:59,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:59,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:30:59,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:30:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T02:30:59,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/a17859e4629343cd974429f3ca2b2a31 2024-11-11T02:30:59,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/131b6b595f514be693e07ba977287f00 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/131b6b595f514be693e07ba977287f00 2024-11-11T02:30:59,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T02:30:59,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:59,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:30:59,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:59,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:59,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:30:59,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/131b6b595f514be693e07ba977287f00, entries=150, sequenceid=132, filesize=11.8 K 2024-11-11T02:30:59,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/929dcd21333347c6838a888735ab9247 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/929dcd21333347c6838a888735ab9247 2024-11-11T02:30:59,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/929dcd21333347c6838a888735ab9247, entries=150, sequenceid=132, filesize=11.8 K 2024-11-11T02:30:59,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/a17859e4629343cd974429f3ca2b2a31 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a17859e4629343cd974429f3ca2b2a31 2024-11-11T02:30:59,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a17859e4629343cd974429f3ca2b2a31, entries=150, sequenceid=132, filesize=11.8 K 2024-11-11T02:30:59,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 21635db8226dc83255808b1cf0db52eb in 1354ms, sequenceid=132, compaction requested=true 2024-11-11T02:30:59,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:30:59,526 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:30:59,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:30:59,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:30:59,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:30:59,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:30:59,527 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:30:59,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:30:59,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:30:59,528 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36241 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:30:59,528 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/A is initiating minor compaction (all files) 2024-11-11T02:30:59,528 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/A in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:59,529 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/3c0aa4b48f5642e3a92507054a676284, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/6718abb40e5e4fe9b98e4f7d44e2b1b4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/131b6b595f514be693e07ba977287f00] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=35.4 K 2024-11-11T02:30:59,529 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36241 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:30:59,529 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c0aa4b48f5642e3a92507054a676284, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731292255380 2024-11-11T02:30:59,529 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/B is initiating minor compaction (all files) 2024-11-11T02:30:59,530 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/B in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:59,530 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/d28e927ddd7c47648f5d6e777edf1d80, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e58149d7bf434c80bae4e8a5f40c1149, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/929dcd21333347c6838a888735ab9247] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=35.4 K 2024-11-11T02:30:59,530 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 6718abb40e5e4fe9b98e4f7d44e2b1b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731292256077 2024-11-11T02:30:59,530 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting d28e927ddd7c47648f5d6e777edf1d80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731292255380 2024-11-11T02:30:59,531 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 131b6b595f514be693e07ba977287f00, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731292257821 2024-11-11T02:30:59,531 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting e58149d7bf434c80bae4e8a5f40c1149, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731292256077 2024-11-11T02:30:59,532 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 929dcd21333347c6838a888735ab9247, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731292257821 2024-11-11T02:30:59,549 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#A#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:30:59,550 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/b48fb388bfc844fbbda9c3310bfa941e is 50, key is test_row_0/A:col10/1731292257821/Put/seqid=0 2024-11-11T02:30:59,556 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#B#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:30:59,556 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/56cd0e269e6440649d1ffff64ec3976f is 50, key is test_row_0/B:col10/1731292257821/Put/seqid=0 2024-11-11T02:30:59,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741858_1034 (size=12341) 2024-11-11T02:30:59,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741859_1035 (size=12341) 2024-11-11T02:30:59,591 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/56cd0e269e6440649d1ffff64ec3976f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/56cd0e269e6440649d1ffff64ec3976f 2024-11-11T02:30:59,604 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/B of 21635db8226dc83255808b1cf0db52eb into 56cd0e269e6440649d1ffff64ec3976f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:30:59,604 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:30:59,604 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/B, priority=13, startTime=1731292259526; duration=0sec 2024-11-11T02:30:59,604 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:30:59,605 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:B 2024-11-11T02:30:59,605 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:30:59,609 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36241 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:30:59,609 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/C is initiating minor compaction (all files) 2024-11-11T02:30:59,609 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/C in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:59,609 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1d7ce294ffe845329b590b22b96fe8b1, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/743117e6d6ae4ef099e790005af0c8a6, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a17859e4629343cd974429f3ca2b2a31] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=35.4 K 2024-11-11T02:30:59,612 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d7ce294ffe845329b590b22b96fe8b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731292255380 2024-11-11T02:30:59,612 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 743117e6d6ae4ef099e790005af0c8a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731292256077 2024-11-11T02:30:59,613 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting a17859e4629343cd974429f3ca2b2a31, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731292257821 2024-11-11T02:30:59,638 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#C#compaction#23 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:30:59,639 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/14ebdaa4735f4c1f9d4d56136f931672 is 50, key is test_row_0/C:col10/1731292257821/Put/seqid=0 2024-11-11T02:30:59,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T02:30:59,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:30:59,644 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T02:30:59,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:30:59,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:30:59,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:30:59,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:30:59,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:30:59,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:30:59,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741860_1036 (size=12341) 2024-11-11T02:30:59,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/c9bcc39f7358429caf91ef01e31cb23c is 50, key is test_row_0/A:col10/1731292258204/Put/seqid=0 2024-11-11T02:30:59,665 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/14ebdaa4735f4c1f9d4d56136f931672 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/14ebdaa4735f4c1f9d4d56136f931672 2024-11-11T02:30:59,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741861_1037 (size=12151) 2024-11-11T02:30:59,678 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/C of 21635db8226dc83255808b1cf0db52eb into 14ebdaa4735f4c1f9d4d56136f931672(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:30:59,678 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:30:59,678 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/C, priority=13, startTime=1731292259527; duration=0sec 2024-11-11T02:30:59,678 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:30:59,678 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:C 2024-11-11T02:30:59,981 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/b48fb388bfc844fbbda9c3310bfa941e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b48fb388bfc844fbbda9c3310bfa941e 2024-11-11T02:30:59,992 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/A of 21635db8226dc83255808b1cf0db52eb into b48fb388bfc844fbbda9c3310bfa941e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:30:59,992 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:30:59,992 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/A, priority=13, startTime=1731292259526; duration=0sec 2024-11-11T02:30:59,993 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:30:59,993 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:A 2024-11-11T02:31:00,073 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/c9bcc39f7358429caf91ef01e31cb23c 2024-11-11T02:31:00,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/a40fbb9d96f34856a02f7ef535f0df36 is 50, key is test_row_0/B:col10/1731292258204/Put/seqid=0 2024-11-11T02:31:00,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741862_1038 (size=12151) 2024-11-11T02:31:00,109 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/a40fbb9d96f34856a02f7ef535f0df36 2024-11-11T02:31:00,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/1854d9f751f1448fb1a147a052e4c1bd is 50, key is test_row_0/C:col10/1731292258204/Put/seqid=0 2024-11-11T02:31:00,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741863_1039 (size=12151) 2024-11-11T02:31:00,148 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/1854d9f751f1448fb1a147a052e4c1bd 2024-11-11T02:31:00,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/c9bcc39f7358429caf91ef01e31cb23c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c9bcc39f7358429caf91ef01e31cb23c 2024-11-11T02:31:00,169 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c9bcc39f7358429caf91ef01e31cb23c, entries=150, sequenceid=142, filesize=11.9 K 2024-11-11T02:31:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/a40fbb9d96f34856a02f7ef535f0df36 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/a40fbb9d96f34856a02f7ef535f0df36 2024-11-11T02:31:00,182 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/a40fbb9d96f34856a02f7ef535f0df36, entries=150, sequenceid=142, filesize=11.9 K 2024-11-11T02:31:00,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/1854d9f751f1448fb1a147a052e4c1bd as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1854d9f751f1448fb1a147a052e4c1bd 2024-11-11T02:31:00,198 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1854d9f751f1448fb1a147a052e4c1bd, entries=150, sequenceid=142, filesize=11.9 K 2024-11-11T02:31:00,200 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 21635db8226dc83255808b1cf0db52eb in 556ms, sequenceid=142, compaction requested=false 2024-11-11T02:31:00,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:00,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:00,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T02:31:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-11T02:31:00,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T02:31:00,210 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9480 sec 2024-11-11T02:31:00,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.9640 sec 2024-11-11T02:31:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:00,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:31:00,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:00,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:00,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:00,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:00,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:00,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:00,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/557a2ad47ede4db289089bb0d72bad61 is 50, key is test_row_0/A:col10/1731292260301/Put/seqid=0 2024-11-11T02:31:00,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741864_1040 (size=14541) 2024-11-11T02:31:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T02:31:00,396 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:00,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731292260353, max timestamp=1731292260364Num uniques 859; 2024-11-11T02:31:00,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:31:00,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731292260353, max timestamp=1731292260376Num uniques 859; 2024-11-11T02:31:00,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:31:00,398 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-11T02:31:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T02:31:00,401 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:00,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1731292260353, max timestamp=1731292260376Num uniques 859; 2024-11-11T02:31:00,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:31:00,402 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:00,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292320430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292320432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292320439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292320442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292320442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T02:31:00,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292320546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292320546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T02:31:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:00,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:00,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:00,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292320555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292320556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292320556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T02:31:00,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:00,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:00,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:00,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:00,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T02:31:00,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/557a2ad47ede4db289089bb0d72bad61 2024-11-11T02:31:00,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292320756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/704a3e93dfbe4387a3165f435b50de1f is 50, key is test_row_0/B:col10/1731292260301/Put/seqid=0 2024-11-11T02:31:00,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292320757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292320766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292320766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292320766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:00,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:00,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741865_1041 (size=12151) 2024-11-11T02:31:00,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/704a3e93dfbe4387a3165f435b50de1f 2024-11-11T02:31:00,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/fb945d69c12c434c968e1913e5855b14 is 50, key is test_row_0/C:col10/1731292260301/Put/seqid=0 2024-11-11T02:31:00,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741866_1042 (size=12151) 2024-11-11T02:31:00,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/fb945d69c12c434c968e1913e5855b14 2024-11-11T02:31:00,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/557a2ad47ede4db289089bb0d72bad61 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/557a2ad47ede4db289089bb0d72bad61 2024-11-11T02:31:00,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/557a2ad47ede4db289089bb0d72bad61, entries=200, sequenceid=154, filesize=14.2 K 2024-11-11T02:31:00,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/704a3e93dfbe4387a3165f435b50de1f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/704a3e93dfbe4387a3165f435b50de1f 2024-11-11T02:31:00,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/704a3e93dfbe4387a3165f435b50de1f, entries=150, sequenceid=154, filesize=11.9 K 2024-11-11T02:31:00,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/fb945d69c12c434c968e1913e5855b14 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/fb945d69c12c434c968e1913e5855b14 2024-11-11T02:31:00,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/fb945d69c12c434c968e1913e5855b14, entries=150, sequenceid=154, filesize=11.9 K 2024-11-11T02:31:00,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 21635db8226dc83255808b1cf0db52eb in 544ms, sequenceid=154, compaction requested=true 2024-11-11T02:31:00,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:00,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:00,847 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:00,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:00,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:00,848 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:00,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:00,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:00,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:00,849 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39033 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:00,849 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/A is initiating minor compaction (all files) 2024-11-11T02:31:00,849 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/A in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:00,849 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b48fb388bfc844fbbda9c3310bfa941e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c9bcc39f7358429caf91ef01e31cb23c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/557a2ad47ede4db289089bb0d72bad61] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=38.1 K 2024-11-11T02:31:00,850 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:00,850 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/B is initiating minor compaction (all files) 2024-11-11T02:31:00,850 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/B in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:00,850 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/56cd0e269e6440649d1ffff64ec3976f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/a40fbb9d96f34856a02f7ef535f0df36, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/704a3e93dfbe4387a3165f435b50de1f] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=35.8 K 2024-11-11T02:31:00,850 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting b48fb388bfc844fbbda9c3310bfa941e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731292257821 2024-11-11T02:31:00,851 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 56cd0e269e6440649d1ffff64ec3976f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731292257821 2024-11-11T02:31:00,851 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9bcc39f7358429caf91ef01e31cb23c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731292258189 2024-11-11T02:31:00,851 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting a40fbb9d96f34856a02f7ef535f0df36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731292258189 2024-11-11T02:31:00,851 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 704a3e93dfbe4387a3165f435b50de1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731292260295 2024-11-11T02:31:00,851 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 557a2ad47ede4db289089bb0d72bad61, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731292260291 2024-11-11T02:31:00,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T02:31:00,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:00,864 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-11T02:31:00,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:00,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:00,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:00,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:00,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:00,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:00,867 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:00,870 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#A#compaction#30 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:00,870 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/0d733a0ba626475aa5e959fe033545ee is 50, key is test_row_0/A:col10/1731292260301/Put/seqid=0 2024-11-11T02:31:00,871 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/b972020d09e24903bd607073f54f6925 is 50, key is test_row_0/B:col10/1731292260301/Put/seqid=0 2024-11-11T02:31:00,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/db79f9f5a9154305b5214dc83df0e591 is 50, key is test_row_0/A:col10/1731292260418/Put/seqid=0 2024-11-11T02:31:00,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741867_1043 (size=12493) 2024-11-11T02:31:00,915 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/0d733a0ba626475aa5e959fe033545ee as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/0d733a0ba626475aa5e959fe033545ee 2024-11-11T02:31:00,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741869_1045 (size=12151) 2024-11-11T02:31:00,919 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/db79f9f5a9154305b5214dc83df0e591 2024-11-11T02:31:00,925 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/A of 21635db8226dc83255808b1cf0db52eb into 0d733a0ba626475aa5e959fe033545ee(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:00,925 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:00,925 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/A, priority=13, startTime=1731292260847; duration=0sec 2024-11-11T02:31:00,925 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:00,925 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:A 2024-11-11T02:31:00,925 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:00,929 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:00,929 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/C is initiating minor compaction (all files) 2024-11-11T02:31:00,929 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/C in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:00,930 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/14ebdaa4735f4c1f9d4d56136f931672, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1854d9f751f1448fb1a147a052e4c1bd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/fb945d69c12c434c968e1913e5855b14] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=35.8 K 2024-11-11T02:31:00,931 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14ebdaa4735f4c1f9d4d56136f931672, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731292257821 2024-11-11T02:31:00,932 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1854d9f751f1448fb1a147a052e4c1bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731292258189 2024-11-11T02:31:00,933 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb945d69c12c434c968e1913e5855b14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731292260295 2024-11-11T02:31:00,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/89244937f4c5424798e26b983c31e885 is 50, key is test_row_0/B:col10/1731292260418/Put/seqid=0 2024-11-11T02:31:00,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741868_1044 (size=12493) 2024-11-11T02:31:00,951 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/b972020d09e24903bd607073f54f6925 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/b972020d09e24903bd607073f54f6925 2024-11-11T02:31:00,955 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#C#compaction#34 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:00,956 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/94c139ea9fcc4fa6b51733412c76d1a0 is 50, key is test_row_0/C:col10/1731292260301/Put/seqid=0 2024-11-11T02:31:00,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741870_1046 (size=12151) 2024-11-11T02:31:00,965 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/89244937f4c5424798e26b983c31e885 2024-11-11T02:31:00,969 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/B of 21635db8226dc83255808b1cf0db52eb into b972020d09e24903bd607073f54f6925(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:00,969 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:00,969 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/B, priority=13, startTime=1731292260848; duration=0sec 2024-11-11T02:31:00,970 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:00,970 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:B 2024-11-11T02:31:00,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/165a35b68e0b4888954cf3aa29936223 is 50, key is test_row_0/C:col10/1731292260418/Put/seqid=0 2024-11-11T02:31:00,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741871_1047 (size=12493) 2024-11-11T02:31:01,013 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/94c139ea9fcc4fa6b51733412c76d1a0 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/94c139ea9fcc4fa6b51733412c76d1a0 2024-11-11T02:31:01,024 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/C of 21635db8226dc83255808b1cf0db52eb into 94c139ea9fcc4fa6b51733412c76d1a0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:01,025 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:01,025 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/C, priority=13, startTime=1731292260848; duration=0sec 2024-11-11T02:31:01,025 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:01,025 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:C 2024-11-11T02:31:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T02:31:01,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741872_1048 (size=12151) 2024-11-11T02:31:01,040 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/165a35b68e0b4888954cf3aa29936223 2024-11-11T02:31:01,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/db79f9f5a9154305b5214dc83df0e591 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/db79f9f5a9154305b5214dc83df0e591 2024-11-11T02:31:01,063 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/db79f9f5a9154305b5214dc83df0e591, entries=150, sequenceid=191, filesize=11.9 K 2024-11-11T02:31:01,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/89244937f4c5424798e26b983c31e885 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/89244937f4c5424798e26b983c31e885 2024-11-11T02:31:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:01,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:01,079 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/89244937f4c5424798e26b983c31e885, entries=150, sequenceid=191, filesize=11.9 K 2024-11-11T02:31:01,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/165a35b68e0b4888954cf3aa29936223 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/165a35b68e0b4888954cf3aa29936223 2024-11-11T02:31:01,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292321082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292321082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292321085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292321084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,094 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/165a35b68e0b4888954cf3aa29936223, entries=150, sequenceid=191, filesize=11.9 K 2024-11-11T02:31:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,095 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 21635db8226dc83255808b1cf0db52eb in 231ms, sequenceid=191, compaction requested=false 2024-11-11T02:31:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T02:31:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-11T02:31:01,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:31:01,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:01,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:01,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:01,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:01,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:01,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:01,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T02:31:01,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 696 msec 2024-11-11T02:31:01,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 712 msec 2024-11-11T02:31:01,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/724886dd04ad4b2cbd494a253dcc0687 is 50, key is test_row_0/A:col10/1731292261079/Put/seqid=0 2024-11-11T02:31:01,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741873_1049 (size=14541) 2024-11-11T02:31:01,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/724886dd04ad4b2cbd494a253dcc0687 2024-11-11T02:31:01,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/e3e4781ee71d49bf997e01928f93e3ac is 50, key is test_row_0/B:col10/1731292261079/Put/seqid=0 2024-11-11T02:31:01,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741874_1050 (size=12151) 2024-11-11T02:31:01,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/e3e4781ee71d49bf997e01928f93e3ac 2024-11-11T02:31:01,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/f55d5ba3ac9640e593782dae277b01ad is 50, key is test_row_0/C:col10/1731292261079/Put/seqid=0 2024-11-11T02:31:01,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292261200, max timestamp=1731292261213Num uniques 859; 2024-11-11T02:31:01,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:31:01,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292261200, max timestamp=1731292261214Num uniques 859; 2024-11-11T02:31:01,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292261200, max timestamp=1731292261214Num uniques 859; 2024-11-11T02:31:01,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:31:01,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:31:01,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741875_1051 (size=12151) 2024-11-11T02:31:01,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/f55d5ba3ac9640e593782dae277b01ad 2024-11-11T02:31:01,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/724886dd04ad4b2cbd494a253dcc0687 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/724886dd04ad4b2cbd494a253dcc0687 2024-11-11T02:31:01,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/724886dd04ad4b2cbd494a253dcc0687, entries=200, sequenceid=204, filesize=14.2 K 2024-11-11T02:31:01,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/e3e4781ee71d49bf997e01928f93e3ac as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e3e4781ee71d49bf997e01928f93e3ac 2024-11-11T02:31:01,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e3e4781ee71d49bf997e01928f93e3ac, entries=150, sequenceid=204, filesize=11.9 K 2024-11-11T02:31:01,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/f55d5ba3ac9640e593782dae277b01ad as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/f55d5ba3ac9640e593782dae277b01ad 2024-11-11T02:31:01,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292321267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292321270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292321270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292321270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/f55d5ba3ac9640e593782dae277b01ad, entries=150, sequenceid=204, filesize=11.9 K 2024-11-11T02:31:01,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 21635db8226dc83255808b1cf0db52eb in 192ms, sequenceid=204, compaction requested=true 2024-11-11T02:31:01,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:01,289 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:01,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:01,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:01,289 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:01,290 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:01,291 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/A is initiating minor compaction (all files) 2024-11-11T02:31:01,291 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/A in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:01,291 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/0d733a0ba626475aa5e959fe033545ee, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/db79f9f5a9154305b5214dc83df0e591, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/724886dd04ad4b2cbd494a253dcc0687] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=38.3 K 2024-11-11T02:31:01,291 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:01,291 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/B is initiating minor compaction (all files) 2024-11-11T02:31:01,291 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/B in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:01,292 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/b972020d09e24903bd607073f54f6925, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/89244937f4c5424798e26b983c31e885, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e3e4781ee71d49bf997e01928f93e3ac] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=35.9 K 2024-11-11T02:31:01,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:01,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:01,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:01,292 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting b972020d09e24903bd607073f54f6925, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731292260295 2024-11-11T02:31:01,293 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d733a0ba626475aa5e959fe033545ee, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731292260295 2024-11-11T02:31:01,293 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 89244937f4c5424798e26b983c31e885, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731292260418 2024-11-11T02:31:01,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:01,293 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting db79f9f5a9154305b5214dc83df0e591, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731292260418 2024-11-11T02:31:01,294 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting e3e4781ee71d49bf997e01928f93e3ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292261079 2024-11-11T02:31:01,294 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 724886dd04ad4b2cbd494a253dcc0687, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292261079 2024-11-11T02:31:01,332 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:01,333 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/609c8602cbad4969ad2a3b2bd3dcdbe2 is 50, key is test_row_0/A:col10/1731292261079/Put/seqid=0 2024-11-11T02:31:01,336 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:01,338 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/65282d4f87c44b11b89053cf7f7f5b35 is 50, key is test_row_0/B:col10/1731292261079/Put/seqid=0 2024-11-11T02:31:01,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741876_1052 (size=12595) 2024-11-11T02:31:01,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741877_1053 (size=12595) 2024-11-11T02:31:01,369 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/609c8602cbad4969ad2a3b2bd3dcdbe2 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/609c8602cbad4969ad2a3b2bd3dcdbe2 2024-11-11T02:31:01,378 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/65282d4f87c44b11b89053cf7f7f5b35 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/65282d4f87c44b11b89053cf7f7f5b35 2024-11-11T02:31:01,388 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/B of 21635db8226dc83255808b1cf0db52eb into 65282d4f87c44b11b89053cf7f7f5b35(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:01,388 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:01,388 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/A of 21635db8226dc83255808b1cf0db52eb into 609c8602cbad4969ad2a3b2bd3dcdbe2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:01,388 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/B, priority=13, startTime=1731292261289; duration=0sec 2024-11-11T02:31:01,388 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:01,388 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/A, priority=13, startTime=1731292261289; duration=0sec 2024-11-11T02:31:01,388 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:01,389 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:B 2024-11-11T02:31:01,389 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:01,389 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:A 2024-11-11T02:31:01,389 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:01,390 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:01,391 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/C is initiating minor compaction (all files) 2024-11-11T02:31:01,391 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/C in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:01,391 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/94c139ea9fcc4fa6b51733412c76d1a0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/165a35b68e0b4888954cf3aa29936223, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/f55d5ba3ac9640e593782dae277b01ad] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=35.9 K 2024-11-11T02:31:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:01,392 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 94c139ea9fcc4fa6b51733412c76d1a0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731292260295 2024-11-11T02:31:01,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-11T02:31:01,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:01,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:31:01,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:01,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:31:01,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:01,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:31:01,395 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 165a35b68e0b4888954cf3aa29936223, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731292260418 2024-11-11T02:31:01,397 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting f55d5ba3ac9640e593782dae277b01ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292261079 2024-11-11T02:31:01,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/d951cd6e1ef242f782fde65d1ea69b45 is 50, key is test_row_0/A:col10/1731292261388/Put/seqid=0 2024-11-11T02:31:01,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292321410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292321412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292321418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292321420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,428 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:01,429 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/75346b3a02d54af7a721eee050be1cd8 is 50, key is test_row_0/C:col10/1731292261079/Put/seqid=0 2024-11-11T02:31:01,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741878_1054 (size=14541) 2024-11-11T02:31:01,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741879_1055 (size=12595) 2024-11-11T02:31:01,459 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/75346b3a02d54af7a721eee050be1cd8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/75346b3a02d54af7a721eee050be1cd8 2024-11-11T02:31:01,473 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/C of 21635db8226dc83255808b1cf0db52eb into 75346b3a02d54af7a721eee050be1cd8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:01,473 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:01,473 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/C, priority=13, startTime=1731292261292; duration=0sec 2024-11-11T02:31:01,473 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:01,473 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:C 2024-11-11T02:31:01,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292321525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292321525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T02:31:01,537 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:01,537 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-11T02:31:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T02:31:01,540 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:01,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292321536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,542 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:01,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292321536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292321606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T02:31:01,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T02:31:01,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:01,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:01,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:01,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:01,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:01,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292321735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292321735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292321746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292321746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:01,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:01,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:01,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/d951cd6e1ef242f782fde65d1ea69b45 2024-11-11T02:31:01,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T02:31:01,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:01,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:01,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:01,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:01,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T02:31:01,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/77f22547351f43988a056d761eb46eca is 50, key is test_row_0/B:col10/1731292261388/Put/seqid=0 2024-11-11T02:31:01,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741880_1056 (size=12151) 2024-11-11T02:31:02,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T02:31:02,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:02,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:02,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:02,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:02,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:02,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292322046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292322046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292322056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292322056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T02:31:02,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:02,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:02,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:02,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:02,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T02:31:02,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/77f22547351f43988a056d761eb46eca 2024-11-11T02:31:02,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/1a648dee4870444c83b28f324d18344d is 50, key is test_row_0/C:col10/1731292261388/Put/seqid=0 2024-11-11T02:31:02,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741881_1057 (size=12151) 2024-11-11T02:31:02,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/1a648dee4870444c83b28f324d18344d 2024-11-11T02:31:02,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T02:31:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:02,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:02,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:02,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/d951cd6e1ef242f782fde65d1ea69b45 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/d951cd6e1ef242f782fde65d1ea69b45 2024-11-11T02:31:02,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/d951cd6e1ef242f782fde65d1ea69b45, entries=200, sequenceid=244, filesize=14.2 K 2024-11-11T02:31:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/77f22547351f43988a056d761eb46eca as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/77f22547351f43988a056d761eb46eca 2024-11-11T02:31:02,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/77f22547351f43988a056d761eb46eca, entries=150, sequenceid=244, filesize=11.9 K 2024-11-11T02:31:02,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/1a648dee4870444c83b28f324d18344d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1a648dee4870444c83b28f324d18344d 2024-11-11T02:31:02,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1a648dee4870444c83b28f324d18344d, entries=150, sequenceid=244, filesize=11.9 K 2024-11-11T02:31:02,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 21635db8226dc83255808b1cf0db52eb in 946ms, sequenceid=244, compaction requested=false 2024-11-11T02:31:02,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:02,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T02:31:02,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:02,466 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T02:31:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/c48fc9b667ec4e3cb7694e2c073e2fcf is 50, key is test_row_0/A:col10/1731292261408/Put/seqid=0 2024-11-11T02:31:02,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741882_1058 (size=12151) 2024-11-11T02:31:02,477 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/c48fc9b667ec4e3cb7694e2c073e2fcf 2024-11-11T02:31:02,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/5d4654a4a5b741dfb357321b40bdc058 is 50, key is test_row_0/B:col10/1731292261408/Put/seqid=0 2024-11-11T02:31:02,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741883_1059 (size=12151) 2024-11-11T02:31:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:02,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:02,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292262584, max timestamp=1731292262593Num uniques 859; 2024-11-11T02:31:02,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292262584, max timestamp=1731292262593Num uniques 859; 2024-11-11T02:31:02,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292262584, max timestamp=1731292262593Num uniques 859; 2024-11-11T02:31:02,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:31:02,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:31:02,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:31:02,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292322627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292322629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292322629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14550, totalHeapSize=33312, min timestamp=1731292262584, max timestamp=1731292262634Num uniques 1018; 2024-11-11T02:31:02,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14550, totalHeapSize=33312, min timestamp=1731292262584, max timestamp=1731292262634Num uniques 1018; 2024-11-11T02:31:02,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14556, totalHeapSize=33312, min timestamp=1731292262584, max timestamp=1731292262634Num uniques 1018; 2024-11-11T02:31:02,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46610, new segment data size=14550, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-11T02:31:02,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46610, new segment data size=14550, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-11T02:31:02,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46616, new segment data size=14556, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-11T02:31:02,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T02:31:02,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292322687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292322688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292322745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292322745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292322746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292322795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292322795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,898 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/5d4654a4a5b741dfb357321b40bdc058 2024-11-11T02:31:02,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/0b76c7210e7445dba07c4a5b095f6fff is 50, key is test_row_0/C:col10/1731292261408/Put/seqid=0 2024-11-11T02:31:02,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741884_1060 (size=12151) 2024-11-11T02:31:02,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292322956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292322956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292322956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:02,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:03,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292323006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:03,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292323006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:03,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:03,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292323266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:03,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292323275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292323275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:03,312 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/0b76c7210e7445dba07c4a5b095f6fff 2024-11-11T02:31:03,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/c48fc9b667ec4e3cb7694e2c073e2fcf as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c48fc9b667ec4e3cb7694e2c073e2fcf 2024-11-11T02:31:03,324 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c48fc9b667ec4e3cb7694e2c073e2fcf, entries=150, sequenceid=254, filesize=11.9 K 2024-11-11T02:31:03,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/5d4654a4a5b741dfb357321b40bdc058 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5d4654a4a5b741dfb357321b40bdc058 2024-11-11T02:31:03,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292323325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:03,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292323325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:03,333 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5d4654a4a5b741dfb357321b40bdc058, entries=150, sequenceid=254, filesize=11.9 K 2024-11-11T02:31:03,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/0b76c7210e7445dba07c4a5b095f6fff as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/0b76c7210e7445dba07c4a5b095f6fff 2024-11-11T02:31:03,341 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/0b76c7210e7445dba07c4a5b095f6fff, entries=150, sequenceid=254, filesize=11.9 K 2024-11-11T02:31:03,342 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for 21635db8226dc83255808b1cf0db52eb in 877ms, sequenceid=254, compaction requested=true 2024-11-11T02:31:03,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:03,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:03,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-11T02:31:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-11T02:31:03,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-11T02:31:03,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8020 sec 2024-11-11T02:31:03,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.8120 sec 2024-11-11T02:31:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T02:31:03,685 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:03,686 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-11T02:31:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T02:31:03,689 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:03,689 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:03,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:03,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=154.31 KB heapSize=429.84 KB 2024-11-11T02:31:03,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:03,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:03,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:03,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:03,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:03,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:03,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/d3de1f21118f4c8585df34c8a6736795 is 50, key is test_row_0/A:col10/1731292263778/Put/seqid=0 2024-11-11T02:31:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T02:31:03,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741885_1061 (size=14741) 2024-11-11T02:31:03,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292323795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:03,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/d3de1f21118f4c8585df34c8a6736795 2024-11-11T02:31:03,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292323797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:03,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292323799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:03,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/f7e60015dd2a4073aa7d414e213c0fbc is 50, key is test_row_0/B:col10/1731292263778/Put/seqid=0 2024-11-11T02:31:03,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741886_1062 (size=12301) 2024-11-11T02:31:03,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/f7e60015dd2a4073aa7d414e213c0fbc 2024-11-11T02:31:03,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/63f50f5aec814e1cb5f9c0911bca9eb3 is 50, key is test_row_0/C:col10/1731292263778/Put/seqid=0 2024-11-11T02:31:03,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292323836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:03,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T02:31:03,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:03,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:03,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741887_1063 (size=12301) 2024-11-11T02:31:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:03,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/63f50f5aec814e1cb5f9c0911bca9eb3 2024-11-11T02:31:03,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292323845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:03,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:03,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:03,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/d3de1f21118f4c8585df34c8a6736795 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/d3de1f21118f4c8585df34c8a6736795 2024-11-11T02:31:03,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/d3de1f21118f4c8585df34c8a6736795, entries=200, sequenceid=308, filesize=14.4 K 2024-11-11T02:31:03,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/f7e60015dd2a4073aa7d414e213c0fbc as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/f7e60015dd2a4073aa7d414e213c0fbc 2024-11-11T02:31:03,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/f7e60015dd2a4073aa7d414e213c0fbc, entries=150, sequenceid=308, filesize=12.0 K 2024-11-11T02:31:03,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/63f50f5aec814e1cb5f9c0911bca9eb3 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/63f50f5aec814e1cb5f9c0911bca9eb3 2024-11-11T02:31:03,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/63f50f5aec814e1cb5f9c0911bca9eb3, entries=150, sequenceid=308, filesize=12.0 K 2024-11-11T02:31:03,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~429.80 KB/440112, currentSize=33.54 KB/34350 for 21635db8226dc83255808b1cf0db52eb in 100ms, sequenceid=308, compaction requested=true 2024-11-11T02:31:03,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:03,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:03,879 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:03,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:03,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:03,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:03,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:03,880 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:03,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:03,881 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:03,881 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54028 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:03,881 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/A is initiating minor compaction (all files) 2024-11-11T02:31:03,881 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/B is initiating minor compaction (all files) 2024-11-11T02:31:03,881 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/B in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:03,881 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/A in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:03,881 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/65282d4f87c44b11b89053cf7f7f5b35, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/77f22547351f43988a056d761eb46eca, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5d4654a4a5b741dfb357321b40bdc058, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/f7e60015dd2a4073aa7d414e213c0fbc] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=48.0 K 2024-11-11T02:31:03,881 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/609c8602cbad4969ad2a3b2bd3dcdbe2, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/d951cd6e1ef242f782fde65d1ea69b45, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c48fc9b667ec4e3cb7694e2c073e2fcf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/d3de1f21118f4c8585df34c8a6736795] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=52.8 K 2024-11-11T02:31:03,882 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 609c8602cbad4969ad2a3b2bd3dcdbe2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292261079 2024-11-11T02:31:03,882 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65282d4f87c44b11b89053cf7f7f5b35, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292261079 2024-11-11T02:31:03,882 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77f22547351f43988a056d761eb46eca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731292261247 2024-11-11T02:31:03,882 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting d951cd6e1ef242f782fde65d1ea69b45, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731292261247 2024-11-11T02:31:03,883 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d4654a4a5b741dfb357321b40bdc058, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731292261408 2024-11-11T02:31:03,883 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting c48fc9b667ec4e3cb7694e2c073e2fcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731292261408 2024-11-11T02:31:03,883 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7e60015dd2a4073aa7d414e213c0fbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731292262677 2024-11-11T02:31:03,883 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting d3de1f21118f4c8585df34c8a6736795, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731292262669 2024-11-11T02:31:03,901 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#B#compaction#51 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:03,902 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/570bf6ba2a8246a4b1aca75d44f77ec8 is 50, key is test_row_0/B:col10/1731292263778/Put/seqid=0 2024-11-11T02:31:03,904 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:03,905 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/ccc3655ca45243efa0ea029a3bb1ab8d is 50, key is test_row_0/A:col10/1731292263778/Put/seqid=0 2024-11-11T02:31:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:03,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:31:03,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:03,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:03,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:03,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:03,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:03,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:03,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741888_1064 (size=12881) 2024-11-11T02:31:03,932 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/570bf6ba2a8246a4b1aca75d44f77ec8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/570bf6ba2a8246a4b1aca75d44f77ec8 2024-11-11T02:31:03,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/e5f4932a987748ffae954dabbcf7a540 is 50, key is test_row_1/A:col10/1731292263907/Put/seqid=0 2024-11-11T02:31:03,944 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/B of 21635db8226dc83255808b1cf0db52eb into 570bf6ba2a8246a4b1aca75d44f77ec8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:03,944 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:03,944 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/B, priority=12, startTime=1731292263879; duration=0sec 2024-11-11T02:31:03,944 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:03,944 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:B 2024-11-11T02:31:03,945 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:03,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741889_1065 (size=12881) 2024-11-11T02:31:03,952 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:03,952 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/C is initiating minor compaction (all files) 2024-11-11T02:31:03,952 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/C in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:03,952 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/75346b3a02d54af7a721eee050be1cd8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1a648dee4870444c83b28f324d18344d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/0b76c7210e7445dba07c4a5b095f6fff, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/63f50f5aec814e1cb5f9c0911bca9eb3] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=48.0 K 2024-11-11T02:31:03,954 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75346b3a02d54af7a721eee050be1cd8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292261079 2024-11-11T02:31:03,954 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a648dee4870444c83b28f324d18344d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731292261247 2024-11-11T02:31:03,955 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b76c7210e7445dba07c4a5b095f6fff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731292261408 2024-11-11T02:31:03,956 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63f50f5aec814e1cb5f9c0911bca9eb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731292262677 2024-11-11T02:31:03,959 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/ccc3655ca45243efa0ea029a3bb1ab8d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/ccc3655ca45243efa0ea029a3bb1ab8d 2024-11-11T02:31:03,974 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#C#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:03,975 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/ea526a211de640c394baa497d5242453 is 50, key is test_row_0/C:col10/1731292263778/Put/seqid=0 2024-11-11T02:31:03,975 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/A of 21635db8226dc83255808b1cf0db52eb into ccc3655ca45243efa0ea029a3bb1ab8d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:03,976 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:03,976 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/A, priority=12, startTime=1731292263879; duration=0sec 2024-11-11T02:31:03,976 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:03,976 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:A 2024-11-11T02:31:03,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741890_1066 (size=9857) 2024-11-11T02:31:03,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292263948, max timestamp=1731292263972Num uniques 859; 2024-11-11T02:31:03,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:31:03,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292263948, max timestamp=1731292263972Num uniques 859; 2024-11-11T02:31:03,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:31:03,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292263948, max timestamp=1731292263972Num uniques 859; 2024-11-11T02:31:03,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:31:03,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/e5f4932a987748ffae954dabbcf7a540 2024-11-11T02:31:03,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T02:31:03,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:03,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:03,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:03,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:03,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:04,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/d73b5bcd49ab41acaf98a661b058ad04 is 50, key is test_row_1/B:col10/1731292263907/Put/seqid=0 2024-11-11T02:31:04,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T02:31:04,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741891_1067 (size=12881) 2024-11-11T02:31:04,014 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/ea526a211de640c394baa497d5242453 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/ea526a211de640c394baa497d5242453 2024-11-11T02:31:04,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741892_1068 (size=9857) 2024-11-11T02:31:04,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/d73b5bcd49ab41acaf98a661b058ad04 2024-11-11T02:31:04,023 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/C of 21635db8226dc83255808b1cf0db52eb into ea526a211de640c394baa497d5242453(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:04,023 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:04,023 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/C, priority=12, startTime=1731292263879; duration=0sec 2024-11-11T02:31:04,024 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:04,024 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:C 2024-11-11T02:31:04,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292324025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292324027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/a472108e7ca7473f8e81b3a39587669f is 50, key is test_row_1/C:col10/1731292263907/Put/seqid=0 2024-11-11T02:31:04,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292324035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741893_1069 (size=9857) 2024-11-11T02:31:04,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292324146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292324146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292324146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T02:31:04,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:04,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:04,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:04,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:04,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:04,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T02:31:04,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:04,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:04,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:04,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:04,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:04,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T02:31:04,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292324356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292324356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292324356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/a472108e7ca7473f8e81b3a39587669f 2024-11-11T02:31:04,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/e5f4932a987748ffae954dabbcf7a540 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/e5f4932a987748ffae954dabbcf7a540 2024-11-11T02:31:04,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T02:31:04,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:04,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:04,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:04,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:04,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:04,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:04,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/e5f4932a987748ffae954dabbcf7a540, entries=100, sequenceid=318, filesize=9.6 K 2024-11-11T02:31:04,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/d73b5bcd49ab41acaf98a661b058ad04 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/d73b5bcd49ab41acaf98a661b058ad04 2024-11-11T02:31:04,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/d73b5bcd49ab41acaf98a661b058ad04, entries=100, sequenceid=318, filesize=9.6 K 2024-11-11T02:31:04,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/a472108e7ca7473f8e81b3a39587669f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a472108e7ca7473f8e81b3a39587669f 2024-11-11T02:31:04,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a472108e7ca7473f8e81b3a39587669f, entries=100, sequenceid=318, filesize=9.6 K 2024-11-11T02:31:04,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 21635db8226dc83255808b1cf0db52eb in 572ms, sequenceid=318, compaction requested=false 2024-11-11T02:31:04,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:04,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T02:31:04,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:04,612 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T02:31:04,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:04,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/c6432bc77ae24b40af41696f7017b45e is 50, key is test_row_0/A:col10/1731292264003/Put/seqid=0 2024-11-11T02:31:04,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741894_1070 (size=12301) 2024-11-11T02:31:04,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:04,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292324675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292324676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292324678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292324785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292324785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292324796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T02:31:04,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292324846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,847 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in 21635db8226dc83255808b1cf0db52eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292324866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,873 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in 21635db8226dc83255808b1cf0db52eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292324995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292324995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:05,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292325005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:05,029 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/c6432bc77ae24b40af41696f7017b45e 2024-11-11T02:31:05,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/e6c7f57d7dff49959627583f5e8d70d2 is 50, key is test_row_0/B:col10/1731292264003/Put/seqid=0 2024-11-11T02:31:05,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741895_1071 (size=12301) 2024-11-11T02:31:05,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292325306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:05,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292325306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:05,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292325315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:05,447 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/e6c7f57d7dff49959627583f5e8d70d2 2024-11-11T02:31:05,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/364811b673c449348626cdb3cf8a9c2e is 50, key is test_row_0/C:col10/1731292264003/Put/seqid=0 2024-11-11T02:31:05,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741896_1072 (size=12301) 2024-11-11T02:31:05,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292325815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:05,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292325816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:05,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292325826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:05,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:05,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:05,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T02:31:05,867 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/364811b673c449348626cdb3cf8a9c2e 2024-11-11T02:31:05,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/c6432bc77ae24b40af41696f7017b45e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c6432bc77ae24b40af41696f7017b45e 2024-11-11T02:31:05,881 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c6432bc77ae24b40af41696f7017b45e, entries=150, sequenceid=357, filesize=12.0 K 2024-11-11T02:31:05,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/e6c7f57d7dff49959627583f5e8d70d2 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e6c7f57d7dff49959627583f5e8d70d2 2024-11-11T02:31:05,889 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e6c7f57d7dff49959627583f5e8d70d2, entries=150, sequenceid=357, filesize=12.0 K 2024-11-11T02:31:05,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/364811b673c449348626cdb3cf8a9c2e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/364811b673c449348626cdb3cf8a9c2e 2024-11-11T02:31:05,896 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/364811b673c449348626cdb3cf8a9c2e, entries=150, sequenceid=357, filesize=12.0 K 2024-11-11T02:31:05,897 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 21635db8226dc83255808b1cf0db52eb in 1285ms, sequenceid=357, compaction requested=true 2024-11-11T02:31:05,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:05,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:05,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-11T02:31:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-11T02:31:05,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T02:31:05,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2090 sec 2024-11-11T02:31:05,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.2170 sec 2024-11-11T02:31:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:06,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T02:31:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:06,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/b8770ee58e394c15aa4a0a8953d11de2 is 50, key is test_row_0/A:col10/1731292266826/Put/seqid=0 2024-11-11T02:31:06,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741897_1073 (size=12301) 2024-11-11T02:31:06,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292266862, max timestamp=1731292266897Num uniques 859; 2024-11-11T02:31:06,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292266862, max timestamp=1731292266897Num uniques 859; 2024-11-11T02:31:06,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292266862, max timestamp=1731292266897Num uniques 859; 2024-11-11T02:31:06,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:31:06,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:31:06,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:31:06,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292326920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:06,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292326926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:06,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:06,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292326928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292326929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:06,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:06,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:06,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:06,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:06,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:06,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:06,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292326933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:06,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:06,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:06,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292327035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292327046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292327046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292327046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292327046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292327246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/b8770ee58e394c15aa4a0a8953d11de2 2024-11-11T02:31:07,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/57e17ea8ada345b6838cc62dbd05b2a3 is 50, key is test_row_0/B:col10/1731292266826/Put/seqid=0 2024-11-11T02:31:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292327255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292327256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292327255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741898_1074 (size=12301) 2024-11-11T02:31:07,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/57e17ea8ada345b6838cc62dbd05b2a3 2024-11-11T02:31:07,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292327266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/72c81f78c3b1481b8158b357447bf85e is 50, key is test_row_0/C:col10/1731292266826/Put/seqid=0 2024-11-11T02:31:07,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741899_1075 (size=12301) 2024-11-11T02:31:07,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/72c81f78c3b1481b8158b357447bf85e 2024-11-11T02:31:07,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/b8770ee58e394c15aa4a0a8953d11de2 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b8770ee58e394c15aa4a0a8953d11de2 2024-11-11T02:31:07,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b8770ee58e394c15aa4a0a8953d11de2, entries=150, sequenceid=368, filesize=12.0 K 2024-11-11T02:31:07,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/57e17ea8ada345b6838cc62dbd05b2a3 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/57e17ea8ada345b6838cc62dbd05b2a3 2024-11-11T02:31:07,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/57e17ea8ada345b6838cc62dbd05b2a3, entries=150, sequenceid=368, filesize=12.0 K 2024-11-11T02:31:07,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/72c81f78c3b1481b8158b357447bf85e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/72c81f78c3b1481b8158b357447bf85e 2024-11-11T02:31:07,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/72c81f78c3b1481b8158b357447bf85e, entries=150, sequenceid=368, filesize=12.0 K 2024-11-11T02:31:07,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 21635db8226dc83255808b1cf0db52eb in 487ms, sequenceid=368, compaction requested=true 2024-11-11T02:31:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:07,315 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:07,315 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:07,316 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:07,316 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/B is initiating minor compaction (all files) 2024-11-11T02:31:07,317 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/B in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:07,317 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:07,317 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/A is initiating minor compaction (all files) 2024-11-11T02:31:07,317 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/570bf6ba2a8246a4b1aca75d44f77ec8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/d73b5bcd49ab41acaf98a661b058ad04, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e6c7f57d7dff49959627583f5e8d70d2, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/57e17ea8ada345b6838cc62dbd05b2a3] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=46.2 K 2024-11-11T02:31:07,317 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/A in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:07,317 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/ccc3655ca45243efa0ea029a3bb1ab8d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/e5f4932a987748ffae954dabbcf7a540, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c6432bc77ae24b40af41696f7017b45e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b8770ee58e394c15aa4a0a8953d11de2] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=46.2 K 2024-11-11T02:31:07,317 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccc3655ca45243efa0ea029a3bb1ab8d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731292262677 2024-11-11T02:31:07,318 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 570bf6ba2a8246a4b1aca75d44f77ec8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731292262677 2024-11-11T02:31:07,318 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5f4932a987748ffae954dabbcf7a540, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731292263907 2024-11-11T02:31:07,318 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting d73b5bcd49ab41acaf98a661b058ad04, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731292263907 2024-11-11T02:31:07,318 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c7f57d7dff49959627583f5e8d70d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731292264003 2024-11-11T02:31:07,318 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6432bc77ae24b40af41696f7017b45e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731292264003 2024-11-11T02:31:07,319 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8770ee58e394c15aa4a0a8953d11de2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731292264673 2024-11-11T02:31:07,319 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e17ea8ada345b6838cc62dbd05b2a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731292264673 2024-11-11T02:31:07,333 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#B#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:07,334 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#A#compaction#63 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:07,334 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/fdaee04f60b7494480aa618e30d08781 is 50, key is test_row_0/B:col10/1731292266826/Put/seqid=0 2024-11-11T02:31:07,334 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/70c1589977d14f42b4ebb2102b1d959a is 50, key is test_row_0/A:col10/1731292266826/Put/seqid=0 2024-11-11T02:31:07,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741901_1077 (size=13017) 2024-11-11T02:31:07,355 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/70c1589977d14f42b4ebb2102b1d959a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/70c1589977d14f42b4ebb2102b1d959a 2024-11-11T02:31:07,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741900_1076 (size=13017) 2024-11-11T02:31:07,368 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/A of 21635db8226dc83255808b1cf0db52eb into 70c1589977d14f42b4ebb2102b1d959a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:07,368 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:07,368 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/A, priority=12, startTime=1731292267315; duration=0sec 2024-11-11T02:31:07,368 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:07,368 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:A 2024-11-11T02:31:07,369 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:07,370 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:07,370 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/C is initiating minor compaction (all files) 2024-11-11T02:31:07,370 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/C in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:07,370 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/ea526a211de640c394baa497d5242453, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a472108e7ca7473f8e81b3a39587669f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/364811b673c449348626cdb3cf8a9c2e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/72c81f78c3b1481b8158b357447bf85e] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=46.2 K 2024-11-11T02:31:07,371 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea526a211de640c394baa497d5242453, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731292262677 2024-11-11T02:31:07,372 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting a472108e7ca7473f8e81b3a39587669f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731292263907 2024-11-11T02:31:07,373 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 364811b673c449348626cdb3cf8a9c2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731292264003 2024-11-11T02:31:07,374 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72c81f78c3b1481b8158b357447bf85e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731292264673 2024-11-11T02:31:07,374 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/fdaee04f60b7494480aa618e30d08781 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/fdaee04f60b7494480aa618e30d08781 2024-11-11T02:31:07,386 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/B of 21635db8226dc83255808b1cf0db52eb into fdaee04f60b7494480aa618e30d08781(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:07,386 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:07,386 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/B, priority=12, startTime=1731292267315; duration=0sec 2024-11-11T02:31:07,387 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:07,387 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:B 2024-11-11T02:31:07,398 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#C#compaction#65 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:07,399 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/89557659f467490eb00169860c4a83b0 is 50, key is test_row_0/C:col10/1731292266826/Put/seqid=0 2024-11-11T02:31:07,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741902_1078 (size=13017) 2024-11-11T02:31:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:07,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T02:31:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:07,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/858e8c401b064d789d00ddd220f40c53 is 50, key is test_row_0/A:col10/1731292266927/Put/seqid=0 2024-11-11T02:31:07,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292327571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292327573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292327575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292327575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292327579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741903_1079 (size=14741) 2024-11-11T02:31:07,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292327685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292327685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292327685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292327696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,812 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/89557659f467490eb00169860c4a83b0 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/89557659f467490eb00169860c4a83b0 2024-11-11T02:31:07,818 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/C of 21635db8226dc83255808b1cf0db52eb into 89557659f467490eb00169860c4a83b0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:07,818 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:07,818 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/C, priority=12, startTime=1731292267315; duration=0sec 2024-11-11T02:31:07,818 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:07,818 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:C 2024-11-11T02:31:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T02:31:07,855 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:07,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-11T02:31:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T02:31:07,858 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:07,858 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:07,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:07,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292327895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292327896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292327896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:07,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292327905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:07,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T02:31:07,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/858e8c401b064d789d00ddd220f40c53 2024-11-11T02:31:07,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/127b3fa54b0342e5b51936287f14fd6a is 50, key is test_row_0/B:col10/1731292266927/Put/seqid=0 2024-11-11T02:31:08,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741904_1080 (size=12301) 2024-11-11T02:31:08,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/127b3fa54b0342e5b51936287f14fd6a 2024-11-11T02:31:08,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T02:31:08,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:08,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:08,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:08,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:08,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:08,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:08,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/a894e698295046e49e713b89d61af098 is 50, key is test_row_0/C:col10/1731292266927/Put/seqid=0 2024-11-11T02:31:08,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741905_1081 (size=12301) 2024-11-11T02:31:08,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292328086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:08,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:08,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T02:31:08,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:08,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:08,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:08,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:08,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T02:31:08,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:08,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292328206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:08,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292328206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:08,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292328206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:08,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292328216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:08,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:08,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T02:31:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:08,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:08,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:08,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:08,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/a894e698295046e49e713b89d61af098 2024-11-11T02:31:08,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/858e8c401b064d789d00ddd220f40c53 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/858e8c401b064d789d00ddd220f40c53 2024-11-11T02:31:08,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/858e8c401b064d789d00ddd220f40c53, entries=200, sequenceid=407, filesize=14.4 K 2024-11-11T02:31:08,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/127b3fa54b0342e5b51936287f14fd6a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/127b3fa54b0342e5b51936287f14fd6a 2024-11-11T02:31:08,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/127b3fa54b0342e5b51936287f14fd6a, entries=150, sequenceid=407, filesize=12.0 K 2024-11-11T02:31:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/a894e698295046e49e713b89d61af098 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a894e698295046e49e713b89d61af098 2024-11-11T02:31:08,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a894e698295046e49e713b89d61af098, entries=150, sequenceid=407, filesize=12.0 K 2024-11-11T02:31:08,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 21635db8226dc83255808b1cf0db52eb in 910ms, sequenceid=407, compaction requested=false 2024-11-11T02:31:08,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:08,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T02:31:08,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:08,473 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:31:08,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:08,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:08,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:08,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:08,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:08,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:08,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/a6f863966a264978b897aaaee8719abf is 50, key is test_row_0/A:col10/1731292267577/Put/seqid=0 2024-11-11T02:31:08,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741906_1082 (size=9857) 2024-11-11T02:31:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T02:31:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:08,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:08,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292268756, max timestamp=1731292268767Num uniques 859; 2024-11-11T02:31:08,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292268756, max timestamp=1731292268767Num uniques 859; 2024-11-11T02:31:08,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292268756, max timestamp=1731292268767Num uniques 859; 2024-11-11T02:31:08,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:31:08,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:31:08,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:31:08,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:08,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292328799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292328795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:08,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:08,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292328802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:08,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:08,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:08,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292328802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:08,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:08,885 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/a6f863966a264978b897aaaee8719abf 2024-11-11T02:31:08,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/cbbc8000748e49949e1d6fcfd0be83cf is 50, key is test_row_0/B:col10/1731292267577/Put/seqid=0 2024-11-11T02:31:08,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741907_1083 (size=9857) 2024-11-11T02:31:08,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292328905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292328906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:08,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292328916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:08,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292328916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T02:31:09,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292329106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:09,110 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in 21635db8226dc83255808b1cf0db52eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:09,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292329115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:09,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292329115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:09,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292329126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292329126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:09,303 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/cbbc8000748e49949e1d6fcfd0be83cf 2024-11-11T02:31:09,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/733f30249e8c41eaa4f5974b444bc260 is 50, key is test_row_0/C:col10/1731292267577/Put/seqid=0 2024-11-11T02:31:09,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741908_1084 (size=9857) 2024-11-11T02:31:09,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292329425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:09,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292329426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:09,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292329436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:09,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292329436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:09,720 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/733f30249e8c41eaa4f5974b444bc260 2024-11-11T02:31:09,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/a6f863966a264978b897aaaee8719abf as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/a6f863966a264978b897aaaee8719abf 2024-11-11T02:31:09,731 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/a6f863966a264978b897aaaee8719abf, entries=100, sequenceid=418, filesize=9.6 K 2024-11-11T02:31:09,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/cbbc8000748e49949e1d6fcfd0be83cf as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/cbbc8000748e49949e1d6fcfd0be83cf 2024-11-11T02:31:09,736 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/cbbc8000748e49949e1d6fcfd0be83cf, entries=100, sequenceid=418, filesize=9.6 K 2024-11-11T02:31:09,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/733f30249e8c41eaa4f5974b444bc260 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/733f30249e8c41eaa4f5974b444bc260 2024-11-11T02:31:09,742 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/733f30249e8c41eaa4f5974b444bc260, entries=100, sequenceid=418, filesize=9.6 K 2024-11-11T02:31:09,743 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 21635db8226dc83255808b1cf0db52eb in 1271ms, sequenceid=418, compaction requested=true 2024-11-11T02:31:09,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:09,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:09,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-11T02:31:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-11T02:31:09,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T02:31:09,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8870 sec 2024-11-11T02:31:09,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.8920 sec 2024-11-11T02:31:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:09,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-11T02:31:09,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:09,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:31:09,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:09,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731292268798, max timestamp=1731292269949Num uniques 1068; 2024-11-11T02:31:09,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:09,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:09,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T02:31:09,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731292268798, max timestamp=1731292269949Num uniques 1068; 2024-11-11T02:31:09,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:09,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T02:31:09,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/62f73d8e2fd94b5b80c0138cfcd5d3fd is 50, key is test_row_0/A:col10/1731292269949/Put/seqid=0 2024-11-11T02:31:09,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741909_1085 (size=12301) 2024-11-11T02:31:09,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/62f73d8e2fd94b5b80c0138cfcd5d3fd 2024-11-11T02:31:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T02:31:10,006 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:10,006 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-11T02:31:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T02:31:10,009 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:10,009 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:10,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:10,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/2b2c5f7965784583a07fdf8026de7bbf is 50, key is test_row_0/B:col10/1731292269949/Put/seqid=0 2024-11-11T02:31:10,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741910_1086 (size=12301) 2024-11-11T02:31:10,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/2b2c5f7965784583a07fdf8026de7bbf 2024-11-11T02:31:10,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292330021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292330024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292330025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292330025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/2c845161d96e417f8f3dacde415a656f is 50, key is test_row_0/C:col10/1731292269949/Put/seqid=0 2024-11-11T02:31:10,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741911_1087 (size=12301) 2024-11-11T02:31:10,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/2c845161d96e417f8f3dacde415a656f 2024-11-11T02:31:10,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/62f73d8e2fd94b5b80c0138cfcd5d3fd as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/62f73d8e2fd94b5b80c0138cfcd5d3fd 2024-11-11T02:31:10,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/62f73d8e2fd94b5b80c0138cfcd5d3fd, entries=150, sequenceid=457, filesize=12.0 K 2024-11-11T02:31:10,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/2b2c5f7965784583a07fdf8026de7bbf as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/2b2c5f7965784583a07fdf8026de7bbf 2024-11-11T02:31:10,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/2b2c5f7965784583a07fdf8026de7bbf, entries=150, sequenceid=457, filesize=12.0 K 2024-11-11T02:31:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/2c845161d96e417f8f3dacde415a656f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/2c845161d96e417f8f3dacde415a656f 2024-11-11T02:31:10,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/2c845161d96e417f8f3dacde415a656f, entries=150, sequenceid=457, filesize=12.0 K 2024-11-11T02:31:10,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~203.75 KB/208640, currentSize=114.05 KB/116790 for 21635db8226dc83255808b1cf0db52eb in 121ms, sequenceid=457, compaction requested=true 2024-11-11T02:31:10,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:10,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:10,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:10,072 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:10,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:10,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:10,072 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:10,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:10,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:10,074 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49916 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:10,074 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:10,074 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/A is initiating minor compaction (all files) 2024-11-11T02:31:10,074 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/B is initiating minor compaction (all files) 2024-11-11T02:31:10,074 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/A in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:10,074 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/B in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:10,074 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/fdaee04f60b7494480aa618e30d08781, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/127b3fa54b0342e5b51936287f14fd6a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/cbbc8000748e49949e1d6fcfd0be83cf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/2b2c5f7965784583a07fdf8026de7bbf] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=46.4 K 2024-11-11T02:31:10,074 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/70c1589977d14f42b4ebb2102b1d959a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/858e8c401b064d789d00ddd220f40c53, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/a6f863966a264978b897aaaee8719abf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/62f73d8e2fd94b5b80c0138cfcd5d3fd] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=48.7 K 2024-11-11T02:31:10,075 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70c1589977d14f42b4ebb2102b1d959a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731292264673 2024-11-11T02:31:10,075 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting fdaee04f60b7494480aa618e30d08781, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731292264673 2024-11-11T02:31:10,075 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 127b3fa54b0342e5b51936287f14fd6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731292266926 2024-11-11T02:31:10,075 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 858e8c401b064d789d00ddd220f40c53, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731292266926 2024-11-11T02:31:10,076 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting cbbc8000748e49949e1d6fcfd0be83cf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731292267566 2024-11-11T02:31:10,076 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6f863966a264978b897aaaee8719abf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731292267566 2024-11-11T02:31:10,076 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b2c5f7965784583a07fdf8026de7bbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731292268800 2024-11-11T02:31:10,076 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62f73d8e2fd94b5b80c0138cfcd5d3fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731292268800 2024-11-11T02:31:10,091 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:10,091 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:10,092 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/2241d3adcb2346afa19940299caf1672 is 50, key is test_row_0/A:col10/1731292269949/Put/seqid=0 2024-11-11T02:31:10,093 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/5e9d5546c97a45178edc78a5a21aad88 is 50, key is test_row_0/B:col10/1731292269949/Put/seqid=0 2024-11-11T02:31:10,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741912_1088 (size=13153) 2024-11-11T02:31:10,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741913_1089 (size=13153) 2024-11-11T02:31:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T02:31:10,124 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/2241d3adcb2346afa19940299caf1672 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/2241d3adcb2346afa19940299caf1672 2024-11-11T02:31:10,128 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/5e9d5546c97a45178edc78a5a21aad88 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5e9d5546c97a45178edc78a5a21aad88 2024-11-11T02:31:10,135 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/A of 21635db8226dc83255808b1cf0db52eb into 2241d3adcb2346afa19940299caf1672(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:10,135 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:10,135 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/A, priority=12, startTime=1731292270072; duration=0sec 2024-11-11T02:31:10,136 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:10,136 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:A 2024-11-11T02:31:10,136 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:10,136 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/B of 21635db8226dc83255808b1cf0db52eb into 5e9d5546c97a45178edc78a5a21aad88(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:10,136 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:10,136 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/B, priority=12, startTime=1731292270072; duration=0sec 2024-11-11T02:31:10,137 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:10,137 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:B 2024-11-11T02:31:10,137 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:10,138 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/C is initiating minor compaction (all files) 2024-11-11T02:31:10,138 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/C in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:10,138 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/89557659f467490eb00169860c4a83b0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a894e698295046e49e713b89d61af098, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/733f30249e8c41eaa4f5974b444bc260, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/2c845161d96e417f8f3dacde415a656f] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=46.4 K 2024-11-11T02:31:10,138 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89557659f467490eb00169860c4a83b0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731292264673 2024-11-11T02:31:10,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:10,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T02:31:10,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:10,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:10,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:10,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:10,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:10,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:10,141 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting a894e698295046e49e713b89d61af098, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731292266926 2024-11-11T02:31:10,142 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 733f30249e8c41eaa4f5974b444bc260, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731292267566 2024-11-11T02:31:10,142 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c845161d96e417f8f3dacde415a656f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731292268800 2024-11-11T02:31:10,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/265d290e0ceb47be87495f88a1c827bf is 50, key is test_row_0/A:col10/1731292270021/Put/seqid=0 2024-11-11T02:31:10,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741914_1090 (size=14741) 2024-11-11T02:31:10,158 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#C#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:10,159 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/a363579d686443cb87fb18f806f087b6 is 50, key is test_row_0/C:col10/1731292269949/Put/seqid=0 2024-11-11T02:31:10,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292330153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741915_1091 (size=13153) 2024-11-11T02:31:10,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292330159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292330159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T02:31:10,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:10,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292330160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:10,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:10,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292330265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292330275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292330275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292330275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T02:31:10,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:10,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:10,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:10,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:10,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T02:31:10,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T02:31:10,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:10,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:10,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:10,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:10,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:10,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292330475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292330495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292330495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292330496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/265d290e0ceb47be87495f88a1c827bf 2024-11-11T02:31:10,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/4a0ca62011cf4304b8fcaff9ba308137 is 50, key is test_row_0/B:col10/1731292270021/Put/seqid=0 2024-11-11T02:31:10,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741916_1092 (size=12301) 2024-11-11T02:31:10,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/4a0ca62011cf4304b8fcaff9ba308137 2024-11-11T02:31:10,571 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/a363579d686443cb87fb18f806f087b6 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a363579d686443cb87fb18f806f087b6 2024-11-11T02:31:10,577 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/C of 21635db8226dc83255808b1cf0db52eb into a363579d686443cb87fb18f806f087b6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:10,577 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:10,577 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/C, priority=12, startTime=1731292270072; duration=0sec 2024-11-11T02:31:10,577 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:10,578 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:C 2024-11-11T02:31:10,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/03a0a451120643e9b3962c0a3b9d9966 is 50, key is test_row_0/C:col10/1731292270021/Put/seqid=0 2024-11-11T02:31:10,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741917_1093 (size=12301) 2024-11-11T02:31:10,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/03a0a451120643e9b3962c0a3b9d9966 2024-11-11T02:31:10,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/265d290e0ceb47be87495f88a1c827bf as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/265d290e0ceb47be87495f88a1c827bf 2024-11-11T02:31:10,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/265d290e0ceb47be87495f88a1c827bf, entries=200, sequenceid=480, filesize=14.4 K 2024-11-11T02:31:10,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/4a0ca62011cf4304b8fcaff9ba308137 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/4a0ca62011cf4304b8fcaff9ba308137 2024-11-11T02:31:10,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/4a0ca62011cf4304b8fcaff9ba308137, entries=150, sequenceid=480, filesize=12.0 K 2024-11-11T02:31:10,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/03a0a451120643e9b3962c0a3b9d9966 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/03a0a451120643e9b3962c0a3b9d9966 2024-11-11T02:31:10,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/03a0a451120643e9b3962c0a3b9d9966, entries=150, sequenceid=480, filesize=12.0 K 2024-11-11T02:31:10,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=67.09 KB/68700 for 21635db8226dc83255808b1cf0db52eb in 486ms, sequenceid=480, compaction requested=false 2024-11-11T02:31:10,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T02:31:10,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:10,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:10,625 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T02:31:10,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:10,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:10,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:10,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:10,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:10,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/8d9de5cbbf3742489dc8e6b91ef65de9 is 50, key is test_row_0/A:col10/1731292270157/Put/seqid=0 2024-11-11T02:31:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T02:31:10,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741918_1094 (size=12301) 2024-11-11T02:31:10,640 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/8d9de5cbbf3742489dc8e6b91ef65de9 2024-11-11T02:31:10,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/abaa9ef756944711b9eef88df69ace28 is 50, key is test_row_0/B:col10/1731292270157/Put/seqid=0 2024-11-11T02:31:10,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741919_1095 (size=12301) 2024-11-11T02:31:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:10,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:10,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292330817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292330818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292330821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292330824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292330936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292330936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292330936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292330936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,062 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/abaa9ef756944711b9eef88df69ace28 2024-11-11T02:31:11,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/30d68c3e40c74f0794ff16c56b9bda28 is 50, key is test_row_0/C:col10/1731292270157/Put/seqid=0 2024-11-11T02:31:11,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741920_1096 (size=12301) 2024-11-11T02:31:11,077 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/30d68c3e40c74f0794ff16c56b9bda28 2024-11-11T02:31:11,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/8d9de5cbbf3742489dc8e6b91ef65de9 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/8d9de5cbbf3742489dc8e6b91ef65de9 2024-11-11T02:31:11,087 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/8d9de5cbbf3742489dc8e6b91ef65de9, entries=150, sequenceid=494, filesize=12.0 K 2024-11-11T02:31:11,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/abaa9ef756944711b9eef88df69ace28 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/abaa9ef756944711b9eef88df69ace28 2024-11-11T02:31:11,093 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/abaa9ef756944711b9eef88df69ace28, entries=150, sequenceid=494, filesize=12.0 K 2024-11-11T02:31:11,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/30d68c3e40c74f0794ff16c56b9bda28 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/30d68c3e40c74f0794ff16c56b9bda28 2024-11-11T02:31:11,099 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/30d68c3e40c74f0794ff16c56b9bda28, entries=150, sequenceid=494, filesize=12.0 K 2024-11-11T02:31:11,100 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 21635db8226dc83255808b1cf0db52eb in 474ms, sequenceid=494, compaction requested=true 2024-11-11T02:31:11,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:11,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:11,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-11T02:31:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-11T02:31:11,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-11T02:31:11,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0910 sec 2024-11-11T02:31:11,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.0980 sec 2024-11-11T02:31:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:11,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T02:31:11,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:11,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/b89cc956c4154d91875892d618fafe0c is 50, key is test_row_0/A:col10/1731292271136/Put/seqid=0 2024-11-11T02:31:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T02:31:11,147 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:11,148 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-11T02:31:11,149 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:11,150 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:11,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T02:31:11,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741921_1097 (size=12301) 2024-11-11T02:31:11,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292331152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292331154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292331155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292331159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292331164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T02:31:11,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292331288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292331288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292331288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292331288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292331288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T02:31:11,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:11,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:11,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:11,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:11,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:11,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T02:31:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:11,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T02:31:11,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292331495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292331495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292331495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292331495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292331496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/b89cc956c4154d91875892d618fafe0c 2024-11-11T02:31:11,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/ea37d78231e6463396c70d5f2d50676f is 50, key is test_row_0/B:col10/1731292271136/Put/seqid=0 2024-11-11T02:31:11,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741922_1098 (size=12301) 2024-11-11T02:31:11,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/ea37d78231e6463396c70d5f2d50676f 2024-11-11T02:31:11,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/40773130b80a4fe39b62096d244709aa is 50, key is test_row_0/C:col10/1731292271136/Put/seqid=0 2024-11-11T02:31:11,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741923_1099 (size=12301) 2024-11-11T02:31:11,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/40773130b80a4fe39b62096d244709aa 2024-11-11T02:31:11,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/b89cc956c4154d91875892d618fafe0c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b89cc956c4154d91875892d618fafe0c 2024-11-11T02:31:11,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b89cc956c4154d91875892d618fafe0c, entries=150, sequenceid=514, filesize=12.0 K 2024-11-11T02:31:11,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/ea37d78231e6463396c70d5f2d50676f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/ea37d78231e6463396c70d5f2d50676f 2024-11-11T02:31:11,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/ea37d78231e6463396c70d5f2d50676f, entries=150, sequenceid=514, filesize=12.0 K 2024-11-11T02:31:11,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/40773130b80a4fe39b62096d244709aa as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/40773130b80a4fe39b62096d244709aa 2024-11-11T02:31:11,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T02:31:11,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:11,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/40773130b80a4fe39b62096d244709aa, entries=150, sequenceid=514, filesize=12.0 K 2024-11-11T02:31:11,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:11,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:11,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:11,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:11,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 21635db8226dc83255808b1cf0db52eb in 476ms, sequenceid=514, compaction requested=true 2024-11-11T02:31:11,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:11,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:11,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:11,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:11,614 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:11,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:11,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21635db8226dc83255808b1cf0db52eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:11,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T02:31:11,614 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:11,616 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:11,617 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/B is initiating minor compaction (all files) 2024-11-11T02:31:11,617 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/B in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:11,617 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5e9d5546c97a45178edc78a5a21aad88, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/4a0ca62011cf4304b8fcaff9ba308137, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/abaa9ef756944711b9eef88df69ace28, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/ea37d78231e6463396c70d5f2d50676f] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=48.9 K 2024-11-11T02:31:11,617 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:11,617 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/A is initiating minor compaction (all files) 2024-11-11T02:31:11,617 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/A in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:11,617 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/2241d3adcb2346afa19940299caf1672, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/265d290e0ceb47be87495f88a1c827bf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/8d9de5cbbf3742489dc8e6b91ef65de9, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b89cc956c4154d91875892d618fafe0c] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=51.3 K 2024-11-11T02:31:11,618 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e9d5546c97a45178edc78a5a21aad88, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731292268800 2024-11-11T02:31:11,619 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2241d3adcb2346afa19940299caf1672, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731292268800 2024-11-11T02:31:11,619 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a0ca62011cf4304b8fcaff9ba308137, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731292270002 2024-11-11T02:31:11,621 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 265d290e0ceb47be87495f88a1c827bf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731292270002 2024-11-11T02:31:11,621 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting abaa9ef756944711b9eef88df69ace28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731292270150 2024-11-11T02:31:11,621 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d9de5cbbf3742489dc8e6b91ef65de9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731292270150 2024-11-11T02:31:11,621 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting ea37d78231e6463396c70d5f2d50676f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731292270814 2024-11-11T02:31:11,622 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting b89cc956c4154d91875892d618fafe0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731292270814 2024-11-11T02:31:11,640 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#B#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:11,640 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/ca71a1fc61b94d68bb0bbd87d8ab4431 is 50, key is test_row_0/B:col10/1731292271136/Put/seqid=0 2024-11-11T02:31:11,646 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:11,647 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/36da7306a0294b24b2532f7d7dcb156d is 50, key is test_row_0/A:col10/1731292271136/Put/seqid=0 2024-11-11T02:31:11,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741924_1100 (size=13289) 2024-11-11T02:31:11,685 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/ca71a1fc61b94d68bb0bbd87d8ab4431 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/ca71a1fc61b94d68bb0bbd87d8ab4431 2024-11-11T02:31:11,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741925_1101 (size=13289) 2024-11-11T02:31:11,695 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/B of 21635db8226dc83255808b1cf0db52eb into ca71a1fc61b94d68bb0bbd87d8ab4431(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:11,695 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:11,695 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/B, priority=12, startTime=1731292271613; duration=0sec 2024-11-11T02:31:11,695 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:11,695 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:B 2024-11-11T02:31:11,695 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:11,697 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:11,697 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 21635db8226dc83255808b1cf0db52eb/C is initiating minor compaction (all files) 2024-11-11T02:31:11,697 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21635db8226dc83255808b1cf0db52eb/C in TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:11,697 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a363579d686443cb87fb18f806f087b6, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/03a0a451120643e9b3962c0a3b9d9966, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/30d68c3e40c74f0794ff16c56b9bda28, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/40773130b80a4fe39b62096d244709aa] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp, totalSize=48.9 K 2024-11-11T02:31:11,698 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting a363579d686443cb87fb18f806f087b6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731292268800 2024-11-11T02:31:11,698 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 03a0a451120643e9b3962c0a3b9d9966, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731292270002 2024-11-11T02:31:11,699 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 30d68c3e40c74f0794ff16c56b9bda28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731292270150 2024-11-11T02:31:11,699 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 40773130b80a4fe39b62096d244709aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731292270814 2024-11-11T02:31:11,711 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21635db8226dc83255808b1cf0db52eb#C#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:11,711 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/ee53a8faefb74ac090440bea87d67473 is 50, key is test_row_0/C:col10/1731292271136/Put/seqid=0 2024-11-11T02:31:11,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741926_1102 (size=13289) 2024-11-11T02:31:11,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T02:31:11,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:11,765 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T02:31:11,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:11,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:11,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:11,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:11,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:11,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:11,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/587f17e0957c4500a2fc349db198c654 is 50, key is test_row_0/A:col10/1731292271158/Put/seqid=0 2024-11-11T02:31:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T02:31:11,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741927_1103 (size=12301) 2024-11-11T02:31:11,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. as already flushing 2024-11-11T02:31:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:11,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292331828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292331829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292331830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292331830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292331831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292331936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292331936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292331945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292331945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292331945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:11,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:11,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,093 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/36da7306a0294b24b2532f7d7dcb156d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/36da7306a0294b24b2532f7d7dcb156d 2024-11-11T02:31:12,100 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/A of 21635db8226dc83255808b1cf0db52eb into 36da7306a0294b24b2532f7d7dcb156d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:12,100 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:12,100 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/A, priority=12, startTime=1731292271613; duration=0sec 2024-11-11T02:31:12,100 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:12,100 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:A 2024-11-11T02:31:12,130 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/ee53a8faefb74ac090440bea87d67473 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/ee53a8faefb74ac090440bea87d67473 2024-11-11T02:31:12,135 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21635db8226dc83255808b1cf0db52eb/C of 21635db8226dc83255808b1cf0db52eb into ee53a8faefb74ac090440bea87d67473(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:12,135 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:12,136 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., storeName=21635db8226dc83255808b1cf0db52eb/C, priority=12, startTime=1731292271614; duration=0sec 2024-11-11T02:31:12,136 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:12,136 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21635db8226dc83255808b1cf0db52eb:C 2024-11-11T02:31:12,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292332145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292332145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292332156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292332156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292332156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,179 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/587f17e0957c4500a2fc349db198c654 2024-11-11T02:31:12,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/fade638c7fab46e388944d9c8ad85cb5 is 50, key is test_row_0/B:col10/1731292271158/Put/seqid=0 2024-11-11T02:31:12,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741928_1104 (size=12301) 2024-11-11T02:31:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T02:31:12,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292332455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292332456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292332466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292332466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292332466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,593 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/fade638c7fab46e388944d9c8ad85cb5 2024-11-11T02:31:12,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/7f1134b72e6f4af4b9041f5e6eed3f57 is 50, key is test_row_0/C:col10/1731292271158/Put/seqid=0 2024-11-11T02:31:12,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741929_1105 (size=12301) 2024-11-11T02:31:12,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60212 deadline: 1731292332965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60176 deadline: 1731292332965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60182 deadline: 1731292332975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731292332976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1731292332976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:13,006 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/7f1134b72e6f4af4b9041f5e6eed3f57 2024-11-11T02:31:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/587f17e0957c4500a2fc349db198c654 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/587f17e0957c4500a2fc349db198c654 2024-11-11T02:31:13,017 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/587f17e0957c4500a2fc349db198c654, entries=150, sequenceid=527, filesize=12.0 K 2024-11-11T02:31:13,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/fade638c7fab46e388944d9c8ad85cb5 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/fade638c7fab46e388944d9c8ad85cb5 2024-11-11T02:31:13,023 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/fade638c7fab46e388944d9c8ad85cb5, entries=150, sequenceid=527, filesize=12.0 K 2024-11-11T02:31:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/7f1134b72e6f4af4b9041f5e6eed3f57 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/7f1134b72e6f4af4b9041f5e6eed3f57 2024-11-11T02:31:13,029 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/7f1134b72e6f4af4b9041f5e6eed3f57, entries=150, sequenceid=527, filesize=12.0 K 2024-11-11T02:31:13,030 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 21635db8226dc83255808b1cf0db52eb in 1265ms, sequenceid=527, compaction requested=false 2024-11-11T02:31:13,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:13,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:13,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-11T02:31:13,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-11T02:31:13,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-11T02:31:13,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-11-11T02:31:13,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.8870 sec 2024-11-11T02:31:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T02:31:13,295 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:13,296 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-11T02:31:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T02:31:13,298 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:13,298 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:13,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T02:31:13,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T02:31:13,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:13,451 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T02:31:13,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:13,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:13,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:13,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:13,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:13,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:13,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/8b26c20647c846f187898a66e68bae7a is 50, key is test_row_0/A:col10/1731292271830/Put/seqid=0 2024-11-11T02:31:13,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741930_1106 (size=12301) 2024-11-11T02:31:13,467 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/8b26c20647c846f187898a66e68bae7a 2024-11-11T02:31:13,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/5c2a9aef42e8439ebeff9dabc7300d3b is 50, key is test_row_0/B:col10/1731292271830/Put/seqid=0 2024-11-11T02:31:13,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741931_1107 (size=12301) 2024-11-11T02:31:13,486 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/5c2a9aef42e8439ebeff9dabc7300d3b 2024-11-11T02:31:13,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/8e8b5ab21b9d492c88a39284ea25dcd1 is 50, key is test_row_0/C:col10/1731292271830/Put/seqid=0 2024-11-11T02:31:13,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741932_1108 (size=12301) 2024-11-11T02:31:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T02:31:13,907 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/8e8b5ab21b9d492c88a39284ea25dcd1 2024-11-11T02:31:13,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/8b26c20647c846f187898a66e68bae7a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/8b26c20647c846f187898a66e68bae7a 2024-11-11T02:31:13,917 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/8b26c20647c846f187898a66e68bae7a, entries=150, sequenceid=549, filesize=12.0 K 2024-11-11T02:31:13,918 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-11T02:31:13,918 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-11T02:31:13,918 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:13,918 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:13,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/5c2a9aef42e8439ebeff9dabc7300d3b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5c2a9aef42e8439ebeff9dabc7300d3b 2024-11-11T02:31:13,920 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-11T02:31:13,920 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:13,923 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,923 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,923 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,924 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-11T02:31:13,924 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:13,924 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,924 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,924 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5c2a9aef42e8439ebeff9dabc7300d3b, entries=150, sequenceid=549, filesize=12.0 K 2024-11-11T02:31:13,924 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:13,924 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,924 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:13,924 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,924 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:13,925 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:13,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/8e8b5ab21b9d492c88a39284ea25dcd1 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/8e8b5ab21b9d492c88a39284ea25dcd1 2024-11-11T02:31:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T02:31:13,930 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/8e8b5ab21b9d492c88a39284ea25dcd1, entries=150, sequenceid=549, filesize=12.0 K 2024-11-11T02:31:13,931 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 21635db8226dc83255808b1cf0db52eb in 480ms, sequenceid=549, compaction requested=true 2024-11-11T02:31:13,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 21635db8226dc83255808b1cf0db52eb: 2024-11-11T02:31:13,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:13,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-11T02:31:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-11T02:31:13,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-11T02:31:13,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 634 msec 2024-11-11T02:31:13,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 638 msec 2024-11-11T02:31:13,978 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-11T02:31:13,978 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:13,978 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,979 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,979 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:13,990 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-11T02:31:13,990 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-11T02:31:13,990 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:13,990 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:13,990 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,990 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,991 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,991 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:13,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:13,997 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-11T02:31:13,997 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-11T02:31:13,998 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:13,998 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:13,998 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,998 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,998 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,998 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:13,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:13,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T02:31:14,438 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:14,438 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T02:31:14,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-11T02:31:14,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-11T02:31:14,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-11T02:31:14,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-11T02:31:14,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-11T02:31:14,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T02:31:14,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2840 2024-11-11T02:31:14,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2867 2024-11-11T02:31:14,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T02:31:14,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1305 2024-11-11T02:31:14,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3915 rows 2024-11-11T02:31:14,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1317 2024-11-11T02:31:14,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3951 rows 2024-11-11T02:31:14,442 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:31:14,443 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:31:14,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:14,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:14,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:14,451 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T02:31:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:14,458 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292274458"}]},"ts":"1731292274458"} 2024-11-11T02:31:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T02:31:14,460 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T02:31:14,460 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T02:31:14,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T02:31:14,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21635db8226dc83255808b1cf0db52eb, UNASSIGN}] 2024-11-11T02:31:14,466 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21635db8226dc83255808b1cf0db52eb, UNASSIGN 2024-11-11T02:31:14,468 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=21635db8226dc83255808b1cf0db52eb, regionState=CLOSING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:31:14,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21635db8226dc83255808b1cf0db52eb, UNASSIGN because future has completed 2024-11-11T02:31:14,470 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T02:31:14,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:31:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T02:31:14,629 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:14,630 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T02:31:14,630 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 21635db8226dc83255808b1cf0db52eb, disabling compactions & flushes 2024-11-11T02:31:14,630 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:14,631 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:14,631 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. after waiting 0 ms 2024-11-11T02:31:14,631 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:14,631 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 21635db8226dc83255808b1cf0db52eb 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T02:31:14,631 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=A 2024-11-11T02:31:14,631 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:14,631 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=B 2024-11-11T02:31:14,631 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:14,632 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21635db8226dc83255808b1cf0db52eb, store=C 2024-11-11T02:31:14,632 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:14,638 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/026671822ccc4a528ea146b3e87b685e is 50, key is test_row_1/A:col10/1731292273987/Put/seqid=0 2024-11-11T02:31:14,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741933_1109 (size=9857) 2024-11-11T02:31:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T02:31:15,044 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/026671822ccc4a528ea146b3e87b685e 2024-11-11T02:31:15,057 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/612025c1ba4e46568991f193b849f19d is 50, key is test_row_1/B:col10/1731292273987/Put/seqid=0 2024-11-11T02:31:15,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741934_1110 (size=9857) 2024-11-11T02:31:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T02:31:15,464 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/612025c1ba4e46568991f193b849f19d 2024-11-11T02:31:15,478 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/eece644b770e4847b2855a258a90a8f2 is 50, key is test_row_1/C:col10/1731292273987/Put/seqid=0 2024-11-11T02:31:15,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741935_1111 (size=9857) 2024-11-11T02:31:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T02:31:15,883 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/eece644b770e4847b2855a258a90a8f2 2024-11-11T02:31:15,893 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/A/026671822ccc4a528ea146b3e87b685e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/026671822ccc4a528ea146b3e87b685e 2024-11-11T02:31:15,900 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/026671822ccc4a528ea146b3e87b685e, entries=100, sequenceid=557, filesize=9.6 K 2024-11-11T02:31:15,901 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/B/612025c1ba4e46568991f193b849f19d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/612025c1ba4e46568991f193b849f19d 2024-11-11T02:31:15,907 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/612025c1ba4e46568991f193b849f19d, entries=100, sequenceid=557, filesize=9.6 K 2024-11-11T02:31:15,908 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/.tmp/C/eece644b770e4847b2855a258a90a8f2 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/eece644b770e4847b2855a258a90a8f2 2024-11-11T02:31:15,912 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/eece644b770e4847b2855a258a90a8f2, entries=100, sequenceid=557, filesize=9.6 K 2024-11-11T02:31:15,913 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 21635db8226dc83255808b1cf0db52eb in 1282ms, sequenceid=557, compaction requested=true 2024-11-11T02:31:15,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/a71dfdc747014683b045c5a77b0389dc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/e5ca1ea7fc95406cb50f7e7b0bc5ba6b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/7460008f54044217ab382ce99670cc38, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/97c9cb2a95164738b4643ff6fb493552, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/3c0aa4b48f5642e3a92507054a676284, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/6718abb40e5e4fe9b98e4f7d44e2b1b4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b48fb388bfc844fbbda9c3310bfa941e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/131b6b595f514be693e07ba977287f00, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c9bcc39f7358429caf91ef01e31cb23c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/557a2ad47ede4db289089bb0d72bad61, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/0d733a0ba626475aa5e959fe033545ee, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/db79f9f5a9154305b5214dc83df0e591, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/724886dd04ad4b2cbd494a253dcc0687, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/609c8602cbad4969ad2a3b2bd3dcdbe2, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/d951cd6e1ef242f782fde65d1ea69b45, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c48fc9b667ec4e3cb7694e2c073e2fcf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/d3de1f21118f4c8585df34c8a6736795, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/ccc3655ca45243efa0ea029a3bb1ab8d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/e5f4932a987748ffae954dabbcf7a540, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c6432bc77ae24b40af41696f7017b45e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/70c1589977d14f42b4ebb2102b1d959a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b8770ee58e394c15aa4a0a8953d11de2, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/858e8c401b064d789d00ddd220f40c53, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/a6f863966a264978b897aaaee8719abf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/2241d3adcb2346afa19940299caf1672, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/62f73d8e2fd94b5b80c0138cfcd5d3fd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/265d290e0ceb47be87495f88a1c827bf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/8d9de5cbbf3742489dc8e6b91ef65de9, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b89cc956c4154d91875892d618fafe0c] to archive 2024-11-11T02:31:15,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:31:15,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/a71dfdc747014683b045c5a77b0389dc to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/a71dfdc747014683b045c5a77b0389dc 2024-11-11T02:31:15,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/e5ca1ea7fc95406cb50f7e7b0bc5ba6b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/e5ca1ea7fc95406cb50f7e7b0bc5ba6b 2024-11-11T02:31:15,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/7460008f54044217ab382ce99670cc38 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/7460008f54044217ab382ce99670cc38 2024-11-11T02:31:15,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/97c9cb2a95164738b4643ff6fb493552 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/97c9cb2a95164738b4643ff6fb493552 2024-11-11T02:31:15,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/3c0aa4b48f5642e3a92507054a676284 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/3c0aa4b48f5642e3a92507054a676284 2024-11-11T02:31:15,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/6718abb40e5e4fe9b98e4f7d44e2b1b4 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/6718abb40e5e4fe9b98e4f7d44e2b1b4 2024-11-11T02:31:15,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b48fb388bfc844fbbda9c3310bfa941e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b48fb388bfc844fbbda9c3310bfa941e 2024-11-11T02:31:15,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/131b6b595f514be693e07ba977287f00 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/131b6b595f514be693e07ba977287f00 2024-11-11T02:31:15,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c9bcc39f7358429caf91ef01e31cb23c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c9bcc39f7358429caf91ef01e31cb23c 2024-11-11T02:31:15,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/557a2ad47ede4db289089bb0d72bad61 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/557a2ad47ede4db289089bb0d72bad61 2024-11-11T02:31:15,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/0d733a0ba626475aa5e959fe033545ee to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/0d733a0ba626475aa5e959fe033545ee 2024-11-11T02:31:15,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/db79f9f5a9154305b5214dc83df0e591 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/db79f9f5a9154305b5214dc83df0e591 2024-11-11T02:31:15,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/724886dd04ad4b2cbd494a253dcc0687 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/724886dd04ad4b2cbd494a253dcc0687 2024-11-11T02:31:15,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/609c8602cbad4969ad2a3b2bd3dcdbe2 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/609c8602cbad4969ad2a3b2bd3dcdbe2 2024-11-11T02:31:15,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/d951cd6e1ef242f782fde65d1ea69b45 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/d951cd6e1ef242f782fde65d1ea69b45 2024-11-11T02:31:15,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c48fc9b667ec4e3cb7694e2c073e2fcf to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c48fc9b667ec4e3cb7694e2c073e2fcf 2024-11-11T02:31:15,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/d3de1f21118f4c8585df34c8a6736795 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/d3de1f21118f4c8585df34c8a6736795 2024-11-11T02:31:15,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/ccc3655ca45243efa0ea029a3bb1ab8d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/ccc3655ca45243efa0ea029a3bb1ab8d 2024-11-11T02:31:15,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/e5f4932a987748ffae954dabbcf7a540 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/e5f4932a987748ffae954dabbcf7a540 2024-11-11T02:31:15,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c6432bc77ae24b40af41696f7017b45e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/c6432bc77ae24b40af41696f7017b45e 2024-11-11T02:31:15,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/70c1589977d14f42b4ebb2102b1d959a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/70c1589977d14f42b4ebb2102b1d959a 2024-11-11T02:31:15,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b8770ee58e394c15aa4a0a8953d11de2 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b8770ee58e394c15aa4a0a8953d11de2 2024-11-11T02:31:15,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/858e8c401b064d789d00ddd220f40c53 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/858e8c401b064d789d00ddd220f40c53 2024-11-11T02:31:15,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/a6f863966a264978b897aaaee8719abf to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/a6f863966a264978b897aaaee8719abf 2024-11-11T02:31:15,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/2241d3adcb2346afa19940299caf1672 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/2241d3adcb2346afa19940299caf1672 2024-11-11T02:31:15,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/62f73d8e2fd94b5b80c0138cfcd5d3fd to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/62f73d8e2fd94b5b80c0138cfcd5d3fd 2024-11-11T02:31:15,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/265d290e0ceb47be87495f88a1c827bf to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/265d290e0ceb47be87495f88a1c827bf 2024-11-11T02:31:15,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/8d9de5cbbf3742489dc8e6b91ef65de9 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/8d9de5cbbf3742489dc8e6b91ef65de9 2024-11-11T02:31:15,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b89cc956c4154d91875892d618fafe0c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/b89cc956c4154d91875892d618fafe0c 2024-11-11T02:31:15,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/c9f0157135014bb2a526cfb94967a35c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/a839992aca6845fdbc061dc0da01ddcf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5f42d83d709a491c99615d327b87e9c3, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/d28e927ddd7c47648f5d6e777edf1d80, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/526392d0053740698bbd25ec19b213e0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e58149d7bf434c80bae4e8a5f40c1149, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/56cd0e269e6440649d1ffff64ec3976f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/929dcd21333347c6838a888735ab9247, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/a40fbb9d96f34856a02f7ef535f0df36, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/b972020d09e24903bd607073f54f6925, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/704a3e93dfbe4387a3165f435b50de1f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/89244937f4c5424798e26b983c31e885, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/65282d4f87c44b11b89053cf7f7f5b35, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e3e4781ee71d49bf997e01928f93e3ac, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/77f22547351f43988a056d761eb46eca, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5d4654a4a5b741dfb357321b40bdc058, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/570bf6ba2a8246a4b1aca75d44f77ec8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/f7e60015dd2a4073aa7d414e213c0fbc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/d73b5bcd49ab41acaf98a661b058ad04, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e6c7f57d7dff49959627583f5e8d70d2, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/fdaee04f60b7494480aa618e30d08781, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/57e17ea8ada345b6838cc62dbd05b2a3, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/127b3fa54b0342e5b51936287f14fd6a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/cbbc8000748e49949e1d6fcfd0be83cf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5e9d5546c97a45178edc78a5a21aad88, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/2b2c5f7965784583a07fdf8026de7bbf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/4a0ca62011cf4304b8fcaff9ba308137, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/abaa9ef756944711b9eef88df69ace28, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/ea37d78231e6463396c70d5f2d50676f] to archive 2024-11-11T02:31:15,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:31:15,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/c9f0157135014bb2a526cfb94967a35c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/c9f0157135014bb2a526cfb94967a35c 2024-11-11T02:31:15,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/a839992aca6845fdbc061dc0da01ddcf to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/a839992aca6845fdbc061dc0da01ddcf 2024-11-11T02:31:15,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5f42d83d709a491c99615d327b87e9c3 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5f42d83d709a491c99615d327b87e9c3 2024-11-11T02:31:15,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/d28e927ddd7c47648f5d6e777edf1d80 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/d28e927ddd7c47648f5d6e777edf1d80 2024-11-11T02:31:15,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/526392d0053740698bbd25ec19b213e0 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/526392d0053740698bbd25ec19b213e0 2024-11-11T02:31:15,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e58149d7bf434c80bae4e8a5f40c1149 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e58149d7bf434c80bae4e8a5f40c1149 2024-11-11T02:31:15,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/56cd0e269e6440649d1ffff64ec3976f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/56cd0e269e6440649d1ffff64ec3976f 2024-11-11T02:31:15,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/929dcd21333347c6838a888735ab9247 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/929dcd21333347c6838a888735ab9247 2024-11-11T02:31:15,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/a40fbb9d96f34856a02f7ef535f0df36 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/a40fbb9d96f34856a02f7ef535f0df36 2024-11-11T02:31:15,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/b972020d09e24903bd607073f54f6925 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/b972020d09e24903bd607073f54f6925 2024-11-11T02:31:15,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/704a3e93dfbe4387a3165f435b50de1f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/704a3e93dfbe4387a3165f435b50de1f 2024-11-11T02:31:15,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/89244937f4c5424798e26b983c31e885 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/89244937f4c5424798e26b983c31e885 2024-11-11T02:31:15,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/65282d4f87c44b11b89053cf7f7f5b35 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/65282d4f87c44b11b89053cf7f7f5b35 2024-11-11T02:31:15,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e3e4781ee71d49bf997e01928f93e3ac to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e3e4781ee71d49bf997e01928f93e3ac 2024-11-11T02:31:15,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/77f22547351f43988a056d761eb46eca to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/77f22547351f43988a056d761eb46eca 2024-11-11T02:31:15,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5d4654a4a5b741dfb357321b40bdc058 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5d4654a4a5b741dfb357321b40bdc058 2024-11-11T02:31:15,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/570bf6ba2a8246a4b1aca75d44f77ec8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/570bf6ba2a8246a4b1aca75d44f77ec8 2024-11-11T02:31:15,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/f7e60015dd2a4073aa7d414e213c0fbc to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/f7e60015dd2a4073aa7d414e213c0fbc 2024-11-11T02:31:15,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/d73b5bcd49ab41acaf98a661b058ad04 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/d73b5bcd49ab41acaf98a661b058ad04 2024-11-11T02:31:15,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e6c7f57d7dff49959627583f5e8d70d2 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/e6c7f57d7dff49959627583f5e8d70d2 2024-11-11T02:31:15,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/fdaee04f60b7494480aa618e30d08781 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/fdaee04f60b7494480aa618e30d08781 2024-11-11T02:31:15,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/57e17ea8ada345b6838cc62dbd05b2a3 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/57e17ea8ada345b6838cc62dbd05b2a3 2024-11-11T02:31:15,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/127b3fa54b0342e5b51936287f14fd6a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/127b3fa54b0342e5b51936287f14fd6a 2024-11-11T02:31:15,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/cbbc8000748e49949e1d6fcfd0be83cf to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/cbbc8000748e49949e1d6fcfd0be83cf 2024-11-11T02:31:16,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5e9d5546c97a45178edc78a5a21aad88 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5e9d5546c97a45178edc78a5a21aad88 2024-11-11T02:31:16,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/2b2c5f7965784583a07fdf8026de7bbf to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/2b2c5f7965784583a07fdf8026de7bbf 2024-11-11T02:31:16,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/4a0ca62011cf4304b8fcaff9ba308137 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/4a0ca62011cf4304b8fcaff9ba308137 2024-11-11T02:31:16,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/abaa9ef756944711b9eef88df69ace28 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/abaa9ef756944711b9eef88df69ace28 2024-11-11T02:31:16,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/ea37d78231e6463396c70d5f2d50676f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/ea37d78231e6463396c70d5f2d50676f 2024-11-11T02:31:16,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/f34882f4d61745e19ac70cac86cf2955, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/ec8c97650b8948fb9666cf337df8f321, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/188b7bff934e4f1c80f647a436612340, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1d7ce294ffe845329b590b22b96fe8b1, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/5dd716b75be046d6ab67bdc435b7d083, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/743117e6d6ae4ef099e790005af0c8a6, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/14ebdaa4735f4c1f9d4d56136f931672, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a17859e4629343cd974429f3ca2b2a31, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1854d9f751f1448fb1a147a052e4c1bd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/94c139ea9fcc4fa6b51733412c76d1a0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/fb945d69c12c434c968e1913e5855b14, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/165a35b68e0b4888954cf3aa29936223, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/75346b3a02d54af7a721eee050be1cd8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/f55d5ba3ac9640e593782dae277b01ad, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1a648dee4870444c83b28f324d18344d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/0b76c7210e7445dba07c4a5b095f6fff, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/ea526a211de640c394baa497d5242453, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/63f50f5aec814e1cb5f9c0911bca9eb3, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a472108e7ca7473f8e81b3a39587669f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/364811b673c449348626cdb3cf8a9c2e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/89557659f467490eb00169860c4a83b0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/72c81f78c3b1481b8158b357447bf85e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a894e698295046e49e713b89d61af098, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/733f30249e8c41eaa4f5974b444bc260, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a363579d686443cb87fb18f806f087b6, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/2c845161d96e417f8f3dacde415a656f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/03a0a451120643e9b3962c0a3b9d9966, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/30d68c3e40c74f0794ff16c56b9bda28, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/40773130b80a4fe39b62096d244709aa] to archive 2024-11-11T02:31:16,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:31:16,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/f34882f4d61745e19ac70cac86cf2955 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/f34882f4d61745e19ac70cac86cf2955 2024-11-11T02:31:16,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/ec8c97650b8948fb9666cf337df8f321 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/ec8c97650b8948fb9666cf337df8f321 2024-11-11T02:31:16,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/188b7bff934e4f1c80f647a436612340 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/188b7bff934e4f1c80f647a436612340 2024-11-11T02:31:16,013 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T02:31:16,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1d7ce294ffe845329b590b22b96fe8b1 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1d7ce294ffe845329b590b22b96fe8b1 2024-11-11T02:31:16,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/5dd716b75be046d6ab67bdc435b7d083 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/5dd716b75be046d6ab67bdc435b7d083 2024-11-11T02:31:16,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/743117e6d6ae4ef099e790005af0c8a6 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/743117e6d6ae4ef099e790005af0c8a6 2024-11-11T02:31:16,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/14ebdaa4735f4c1f9d4d56136f931672 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/14ebdaa4735f4c1f9d4d56136f931672 2024-11-11T02:31:16,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a17859e4629343cd974429f3ca2b2a31 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a17859e4629343cd974429f3ca2b2a31 2024-11-11T02:31:16,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1854d9f751f1448fb1a147a052e4c1bd to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1854d9f751f1448fb1a147a052e4c1bd 2024-11-11T02:31:16,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/94c139ea9fcc4fa6b51733412c76d1a0 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/94c139ea9fcc4fa6b51733412c76d1a0 2024-11-11T02:31:16,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/fb945d69c12c434c968e1913e5855b14 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/fb945d69c12c434c968e1913e5855b14 2024-11-11T02:31:16,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/165a35b68e0b4888954cf3aa29936223 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/165a35b68e0b4888954cf3aa29936223 2024-11-11T02:31:16,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/75346b3a02d54af7a721eee050be1cd8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/75346b3a02d54af7a721eee050be1cd8 2024-11-11T02:31:16,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/f55d5ba3ac9640e593782dae277b01ad to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/f55d5ba3ac9640e593782dae277b01ad 2024-11-11T02:31:16,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1a648dee4870444c83b28f324d18344d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/1a648dee4870444c83b28f324d18344d 2024-11-11T02:31:16,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/0b76c7210e7445dba07c4a5b095f6fff to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/0b76c7210e7445dba07c4a5b095f6fff 2024-11-11T02:31:16,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/ea526a211de640c394baa497d5242453 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/ea526a211de640c394baa497d5242453 2024-11-11T02:31:16,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/63f50f5aec814e1cb5f9c0911bca9eb3 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/63f50f5aec814e1cb5f9c0911bca9eb3 2024-11-11T02:31:16,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a472108e7ca7473f8e81b3a39587669f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a472108e7ca7473f8e81b3a39587669f 2024-11-11T02:31:16,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/364811b673c449348626cdb3cf8a9c2e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/364811b673c449348626cdb3cf8a9c2e 2024-11-11T02:31:16,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/89557659f467490eb00169860c4a83b0 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/89557659f467490eb00169860c4a83b0 2024-11-11T02:31:16,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/72c81f78c3b1481b8158b357447bf85e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/72c81f78c3b1481b8158b357447bf85e 2024-11-11T02:31:16,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a894e698295046e49e713b89d61af098 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a894e698295046e49e713b89d61af098 2024-11-11T02:31:16,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/733f30249e8c41eaa4f5974b444bc260 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/733f30249e8c41eaa4f5974b444bc260 2024-11-11T02:31:16,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a363579d686443cb87fb18f806f087b6 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/a363579d686443cb87fb18f806f087b6 2024-11-11T02:31:16,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/2c845161d96e417f8f3dacde415a656f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/2c845161d96e417f8f3dacde415a656f 2024-11-11T02:31:16,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/03a0a451120643e9b3962c0a3b9d9966 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/03a0a451120643e9b3962c0a3b9d9966 2024-11-11T02:31:16,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/30d68c3e40c74f0794ff16c56b9bda28 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/30d68c3e40c74f0794ff16c56b9bda28 2024-11-11T02:31:16,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/40773130b80a4fe39b62096d244709aa to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/40773130b80a4fe39b62096d244709aa 2024-11-11T02:31:16,047 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/recovered.edits/560.seqid, newMaxSeqId=560, maxSeqId=1 2024-11-11T02:31:16,049 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. 2024-11-11T02:31:16,050 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 21635db8226dc83255808b1cf0db52eb: Waiting for close lock at 1731292274630Running coprocessor pre-close hooks at 1731292274630Disabling compacts and flushes for region at 1731292274630Disabling writes for close at 1731292274631 (+1 ms)Obtaining lock to block concurrent updates at 1731292274631Preparing flush snapshotting stores in 21635db8226dc83255808b1cf0db52eb at 1731292274631Finished memstore snapshotting TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731292274632 (+1 ms)Flushing stores of TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb. at 1731292274632Flushing 21635db8226dc83255808b1cf0db52eb/A: creating writer at 1731292274633 (+1 ms)Flushing 21635db8226dc83255808b1cf0db52eb/A: appending metadata at 1731292274637 (+4 ms)Flushing 21635db8226dc83255808b1cf0db52eb/A: closing flushed file at 1731292274637Flushing 21635db8226dc83255808b1cf0db52eb/B: creating writer at 1731292275053 (+416 ms)Flushing 21635db8226dc83255808b1cf0db52eb/B: appending metadata at 1731292275056 (+3 ms)Flushing 21635db8226dc83255808b1cf0db52eb/B: closing flushed file at 1731292275056Flushing 21635db8226dc83255808b1cf0db52eb/C: creating writer at 1731292275474 (+418 ms)Flushing 21635db8226dc83255808b1cf0db52eb/C: appending metadata at 1731292275477 (+3 ms)Flushing 21635db8226dc83255808b1cf0db52eb/C: closing flushed file at 1731292275477Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59eb0bff: reopening flushed file at 1731292275892 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bf5de2a: reopening flushed file at 1731292275901 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8f99a74: reopening flushed file at 1731292275907 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 21635db8226dc83255808b1cf0db52eb in 1282ms, sequenceid=557, compaction requested=true at 1731292275913 (+6 ms)Writing region close event to WAL at 1731292276043 (+130 ms)Running coprocessor post-close hooks at 1731292276047 (+4 ms)Closed at 1731292276049 (+2 ms) 2024-11-11T02:31:16,052 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:16,053 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=21635db8226dc83255808b1cf0db52eb, regionState=CLOSED 2024-11-11T02:31:16,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:31:16,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-11T02:31:16,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 21635db8226dc83255808b1cf0db52eb, server=370bc2ade342,38037,1731292247443 in 1.5850 sec 2024-11-11T02:31:16,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-11T02:31:16,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21635db8226dc83255808b1cf0db52eb, UNASSIGN in 1.5930 sec 2024-11-11T02:31:16,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-11T02:31:16,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5990 sec 2024-11-11T02:31:16,063 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292276063"}]},"ts":"1731292276063"} 2024-11-11T02:31:16,065 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T02:31:16,065 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T02:31:16,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6130 sec 2024-11-11T02:31:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T02:31:16,606 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:16,612 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T02:31:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:16,621 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T02:31:16,622 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:16,627 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:16,631 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/recovered.edits] 2024-11-11T02:31:16,634 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/026671822ccc4a528ea146b3e87b685e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/026671822ccc4a528ea146b3e87b685e 2024-11-11T02:31:16,636 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/36da7306a0294b24b2532f7d7dcb156d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/36da7306a0294b24b2532f7d7dcb156d 2024-11-11T02:31:16,638 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/587f17e0957c4500a2fc349db198c654 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/587f17e0957c4500a2fc349db198c654 2024-11-11T02:31:16,639 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/8b26c20647c846f187898a66e68bae7a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/A/8b26c20647c846f187898a66e68bae7a 2024-11-11T02:31:16,642 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5c2a9aef42e8439ebeff9dabc7300d3b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/5c2a9aef42e8439ebeff9dabc7300d3b 2024-11-11T02:31:16,643 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/612025c1ba4e46568991f193b849f19d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/612025c1ba4e46568991f193b849f19d 2024-11-11T02:31:16,644 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/ca71a1fc61b94d68bb0bbd87d8ab4431 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/ca71a1fc61b94d68bb0bbd87d8ab4431 2024-11-11T02:31:16,645 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/fade638c7fab46e388944d9c8ad85cb5 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/B/fade638c7fab46e388944d9c8ad85cb5 2024-11-11T02:31:16,647 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/7f1134b72e6f4af4b9041f5e6eed3f57 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/7f1134b72e6f4af4b9041f5e6eed3f57 2024-11-11T02:31:16,648 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/8e8b5ab21b9d492c88a39284ea25dcd1 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/8e8b5ab21b9d492c88a39284ea25dcd1 2024-11-11T02:31:16,649 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/ee53a8faefb74ac090440bea87d67473 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/ee53a8faefb74ac090440bea87d67473 2024-11-11T02:31:16,650 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/eece644b770e4847b2855a258a90a8f2 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/C/eece644b770e4847b2855a258a90a8f2 2024-11-11T02:31:16,652 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/recovered.edits/560.seqid to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb/recovered.edits/560.seqid 2024-11-11T02:31:16,652 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/21635db8226dc83255808b1cf0db52eb 2024-11-11T02:31:16,652 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T02:31:16,657 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:16,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-11T02:31:16,669 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T02:31:16,700 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T02:31:16,701 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:16,701 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T02:31:16,701 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731292276701"}]},"ts":"9223372036854775807"} 2024-11-11T02:31:16,705 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T02:31:16,706 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 21635db8226dc83255808b1cf0db52eb, NAME => 'TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T02:31:16,706 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T02:31:16,706 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731292276706"}]},"ts":"9223372036854775807"} 2024-11-11T02:31:16,709 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T02:31:16,710 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:16,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 97 msec 2024-11-11T02:31:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T02:31:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T02:31:16,726 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:16,740 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1958452880_22 at /127.0.0.1:43424 [Waiting for operation #318] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2002917628_22 at /127.0.0.1:38390 [Waiting for operation #310] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/370bc2ade342:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;370bc2ade342:38037-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=973 (was 1135), ProcessCount=11 (was 11), AvailableMemoryMB=5257 (was 5797) 2024-11-11T02:31:16,750 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=973, ProcessCount=11, AvailableMemoryMB=5257 2024-11-11T02:31:16,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T02:31:16,753 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:31:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:16,756 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T02:31:16,756 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:16,756 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-11T02:31:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T02:31:16,757 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T02:31:16,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741936_1112 (size=1061) 2024-11-11T02:31:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T02:31:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T02:31:17,169 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe 2024-11-11T02:31:17,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741937_1113 (size=53) 2024-11-11T02:31:17,210 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T02:31:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T02:31:17,584 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:31:17,584 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e33fc31e962e33bc26f57b81ef4a0f99, disabling compactions & flushes 2024-11-11T02:31:17,584 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:17,584 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:17,585 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. after waiting 0 ms 2024-11-11T02:31:17,585 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:17,585 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:17,585 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e33fc31e962e33bc26f57b81ef4a0f99: Waiting for close lock at 1731292277584Disabling compacts and flushes for region at 1731292277584Disabling writes for close at 1731292277585 (+1 ms)Writing region close event to WAL at 1731292277585Closed at 1731292277585 2024-11-11T02:31:17,588 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T02:31:17,589 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731292277588"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731292277588"}]},"ts":"1731292277588"} 2024-11-11T02:31:17,591 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T02:31:17,593 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T02:31:17,593 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292277593"}]},"ts":"1731292277593"} 2024-11-11T02:31:17,595 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T02:31:17,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, ASSIGN}] 2024-11-11T02:31:17,597 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, ASSIGN 2024-11-11T02:31:17,598 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, ASSIGN; state=OFFLINE, location=370bc2ade342,38037,1731292247443; forceNewPlan=false, retain=false 2024-11-11T02:31:17,749 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=e33fc31e962e33bc26f57b81ef4a0f99, regionState=OPENING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:31:17,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, ASSIGN because future has completed 2024-11-11T02:31:17,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:31:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T02:31:17,916 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:17,917 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:31:17,918 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:17,918 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:31:17,918 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:17,919 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:17,921 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:17,923 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:31:17,923 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e33fc31e962e33bc26f57b81ef4a0f99 columnFamilyName A 2024-11-11T02:31:17,923 DEBUG [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:17,924 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.HStore(327): Store=e33fc31e962e33bc26f57b81ef4a0f99/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:31:17,924 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:17,925 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:31:17,926 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e33fc31e962e33bc26f57b81ef4a0f99 columnFamilyName B 2024-11-11T02:31:17,926 DEBUG [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:17,926 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.HStore(327): Store=e33fc31e962e33bc26f57b81ef4a0f99/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:31:17,926 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:17,927 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:31:17,928 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e33fc31e962e33bc26f57b81ef4a0f99 columnFamilyName C 2024-11-11T02:31:17,928 DEBUG [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:17,928 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.HStore(327): Store=e33fc31e962e33bc26f57b81ef4a0f99/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:31:17,928 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:17,928 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:17,929 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:17,930 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:17,931 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:17,931 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:17,931 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:31:17,932 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:17,935 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:31:17,935 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened e33fc31e962e33bc26f57b81ef4a0f99; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63376478, jitterRate=-0.05561688542366028}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:31:17,935 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:17,936 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for e33fc31e962e33bc26f57b81ef4a0f99: Running coprocessor pre-open hook at 1731292277919Writing region info on filesystem at 1731292277919Initializing all the Stores at 1731292277921 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292277921Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292277921Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292277921Cleaning up temporary data from old regions at 1731292277931 (+10 ms)Running coprocessor post-open hooks at 1731292277935 (+4 ms)Region opened successfully at 1731292277936 (+1 ms) 2024-11-11T02:31:17,937 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., pid=34, masterSystemTime=1731292277908 2024-11-11T02:31:17,939 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:17,939 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:17,940 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=e33fc31e962e33bc26f57b81ef4a0f99, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:31:17,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:31:17,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-11T02:31:17,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 in 189 msec 2024-11-11T02:31:17,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-11T02:31:17,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, ASSIGN in 350 msec 2024-11-11T02:31:17,948 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T02:31:17,948 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292277948"}]},"ts":"1731292277948"} 2024-11-11T02:31:17,950 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T02:31:17,951 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T02:31:17,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1980 sec 2024-11-11T02:31:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T02:31:18,906 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:18,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:18,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:18,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:18,911 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:18,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:18,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:18,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@743a41ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:18,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:18,913 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a13e1f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:18,914 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:18,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:18,919 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T02:31:18,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 370bc2ade342,42785,1731292246873 2024-11-11T02:31:18,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6479eaa0 2024-11-11T02:31:18,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T02:31:18,921 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T02:31:18,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T02:31:18,927 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:31:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:18,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741938_1114 (size=1097) 2024-11-11T02:31:19,350 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T02:31:19,350 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T02:31:19,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T02:31:19,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, REOPEN/MOVE}] 2024-11-11T02:31:19,366 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, REOPEN/MOVE 2024-11-11T02:31:19,367 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=e33fc31e962e33bc26f57b81ef4a0f99, regionState=CLOSING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:31:19,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, REOPEN/MOVE because future has completed 2024-11-11T02:31:19,369 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T02:31:19,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:31:19,521 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,522 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T02:31:19,522 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing e33fc31e962e33bc26f57b81ef4a0f99, disabling compactions & flushes 2024-11-11T02:31:19,522 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:19,522 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:19,522 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. after waiting 0 ms 2024-11-11T02:31:19,522 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:19,526 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T02:31:19,527 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:19,527 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for e33fc31e962e33bc26f57b81ef4a0f99: Waiting for close lock at 1731292279522Running coprocessor pre-close hooks at 1731292279522Disabling compacts and flushes for region at 1731292279522Disabling writes for close at 1731292279522Writing region close event to WAL at 1731292279522Running coprocessor post-close hooks at 1731292279527 (+5 ms)Closed at 1731292279527 2024-11-11T02:31:19,527 WARN [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: e33fc31e962e33bc26f57b81ef4a0f99 to self. 2024-11-11T02:31:19,530 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,531 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=e33fc31e962e33bc26f57b81ef4a0f99, regionState=CLOSED 2024-11-11T02:31:19,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:31:19,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-11T02:31:19,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 in 165 msec 2024-11-11T02:31:19,537 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, REOPEN/MOVE; state=CLOSED, location=370bc2ade342,38037,1731292247443; forceNewPlan=false, retain=true 2024-11-11T02:31:19,688 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=e33fc31e962e33bc26f57b81ef4a0f99, regionState=OPENING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:31:19,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, REOPEN/MOVE because future has completed 2024-11-11T02:31:19,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:31:19,850 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:19,851 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:31:19,851 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,852 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:31:19,852 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,852 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,854 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,855 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:31:19,859 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e33fc31e962e33bc26f57b81ef4a0f99 columnFamilyName A 2024-11-11T02:31:19,861 DEBUG [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:19,862 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.HStore(327): Store=e33fc31e962e33bc26f57b81ef4a0f99/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:31:19,862 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,863 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:31:19,863 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e33fc31e962e33bc26f57b81ef4a0f99 columnFamilyName B 2024-11-11T02:31:19,863 DEBUG [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:19,864 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.HStore(327): Store=e33fc31e962e33bc26f57b81ef4a0f99/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:31:19,864 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,864 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:31:19,864 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e33fc31e962e33bc26f57b81ef4a0f99 columnFamilyName C 2024-11-11T02:31:19,864 DEBUG [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:19,865 INFO [StoreOpener-e33fc31e962e33bc26f57b81ef4a0f99-1 {}] regionserver.HStore(327): Store=e33fc31e962e33bc26f57b81ef4a0f99/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:31:19,865 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:19,865 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,866 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,867 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,868 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,868 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,868 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:31:19,869 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,870 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened e33fc31e962e33bc26f57b81ef4a0f99; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68544533, jitterRate=0.02139313519001007}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:31:19,870 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,871 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for e33fc31e962e33bc26f57b81ef4a0f99: Running coprocessor pre-open hook at 1731292279852Writing region info on filesystem at 1731292279852Initializing all the Stores at 1731292279853 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292279853Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292279853Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292279854 (+1 ms)Cleaning up temporary data from old regions at 1731292279868 (+14 ms)Running coprocessor post-open hooks at 1731292279870 (+2 ms)Region opened successfully at 1731292279871 (+1 ms) 2024-11-11T02:31:19,872 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., pid=39, masterSystemTime=1731292279844 2024-11-11T02:31:19,873 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:19,874 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:19,874 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=e33fc31e962e33bc26f57b81ef4a0f99, regionState=OPEN, openSeqNum=5, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:31:19,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:31:19,876 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42785 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=370bc2ade342,38037,1731292247443, table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T02:31:19,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-11T02:31:19,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 in 185 msec 2024-11-11T02:31:19,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-11T02:31:19,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, REOPEN/MOVE in 513 msec 2024-11-11T02:31:19,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-11T02:31:19,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 523 msec 2024-11-11T02:31:19,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 955 msec 2024-11-11T02:31:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-11T02:31:19,888 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:19,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:19,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:19,890 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:19,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:19,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:19,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60d58df0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:19,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:19,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:19,891 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:19,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5b1922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:19,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:19,894 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d098d10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:19,895 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:19,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328a3c19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:19,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:19,898 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@662e4127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:19,899 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:19,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f2241c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:19,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:19,901 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6016a565, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:19,903 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:19,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47436927, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,904 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:19,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:19,905 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:19,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:19,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:19,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ca5dd39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:19,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:19,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:19,907 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:19,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f38f96c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:19,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:19,909 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6721001b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:19,910 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7272621b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,911 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:19,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:19,912 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@762206d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:19,914 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:19,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65d0ad0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:19,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:19,916 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c10ced, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:19,917 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:19,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a52b838, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:19,918 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:19,919 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:19,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:19,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:19,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d558122, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:19,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:19,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:19,921 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:19,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2ecc06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:19,922 DEBUG [Thread-597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:19,922 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:19,922 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:19,922 DEBUG [Thread-590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:19,922 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:19,923 DEBUG [Thread-588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:19,923 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-11T02:31:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T02:31:19,924 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:19,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:19,925 DEBUG [Thread-601 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:19,925 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:19,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:19,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:19,926 DEBUG [Thread-594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:19,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:19,927 DEBUG [Thread-601 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:19,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:19,927 DEBUG [Thread-592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:19,928 DEBUG [Thread-586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:19,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:19,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:19,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:31:19,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:19,930 DEBUG [Thread-586 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:19,930 DEBUG [Thread-586 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:19,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:19,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:19,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:31:19,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:31:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:31:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:31:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:31:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:31:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:31:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:31:19,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T02:31:19,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:19,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:19,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:19,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:19,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:19,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:20,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111033912d764f64d789dcc2ed7633fb25f_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292279953/Put/seqid=0 2024-11-11T02:31:20,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741939_1115 (size=12154) 2024-11-11T02:31:20,014 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:20,020 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111033912d764f64d789dcc2ed7633fb25f_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111033912d764f64d789dcc2ed7633fb25f_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:20,021 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/d85fe9679127423c82bc55420716f7d4, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T02:31:20,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/d85fe9679127423c82bc55420716f7d4 is 175, key is test_row_0/A:col10/1731292279953/Put/seqid=0 2024-11-11T02:31:20,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292280000, max timestamp=1731292280024Num uniques 859; 2024-11-11T02:31:20,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:31:20,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292280000, max timestamp=1731292280026Num uniques 859; 2024-11-11T02:31:20,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:31:20,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292280000, max timestamp=1731292280024Num uniques 859; 2024-11-11T02:31:20,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:31:20,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741940_1116 (size=30955) 2024-11-11T02:31:20,044 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/d85fe9679127423c82bc55420716f7d4 2024-11-11T02:31:20,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T02:31:20,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:20,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:20,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:20,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292340057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292340086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292340088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292340088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/28fbcd689a3f43b7a110673ea3093d82 is 50, key is test_row_0/B:col10/1731292279953/Put/seqid=0 2024-11-11T02:31:20,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292340090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741941_1117 (size=12001) 2024-11-11T02:31:20,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292340196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292340206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292340206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292340206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292340206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T02:31:20,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:20,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:20,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:20,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T02:31:20,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T02:31:20,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:20,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:20,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:20,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292340405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292340436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292340436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292340436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292340436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/28fbcd689a3f43b7a110673ea3093d82 2024-11-11T02:31:20,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T02:31:20,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:20,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:20,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:20,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/114095b410394c2ab2a4fa3fef7c6f4d is 50, key is test_row_0/C:col10/1731292279953/Put/seqid=0 2024-11-11T02:31:20,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T02:31:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741942_1118 (size=12001) 2024-11-11T02:31:20,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T02:31:20,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:20,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:20,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:20,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292340726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292340745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292340746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292340746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292340756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:20,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T02:31:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:20,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:20,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/114095b410394c2ab2a4fa3fef7c6f4d 2024-11-11T02:31:20,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/d85fe9679127423c82bc55420716f7d4 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/d85fe9679127423c82bc55420716f7d4 2024-11-11T02:31:20,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/d85fe9679127423c82bc55420716f7d4, entries=150, sequenceid=15, filesize=30.2 K 2024-11-11T02:31:20,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/28fbcd689a3f43b7a110673ea3093d82 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/28fbcd689a3f43b7a110673ea3093d82 2024-11-11T02:31:20,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/28fbcd689a3f43b7a110673ea3093d82, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T02:31:20,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/114095b410394c2ab2a4fa3fef7c6f4d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/114095b410394c2ab2a4fa3fef7c6f4d 2024-11-11T02:31:20,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/114095b410394c2ab2a4fa3fef7c6f4d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T02:31:20,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for e33fc31e962e33bc26f57b81ef4a0f99 in 1037ms, sequenceid=15, compaction requested=false 2024-11-11T02:31:20,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:21,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T02:31:21,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:21,005 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-11T02:31:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:21,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111100d8c882c6b84e83ae77895ad31abf2a_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292280085/Put/seqid=0 2024-11-11T02:31:21,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741943_1119 (size=12154) 2024-11-11T02:31:21,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,040 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111100d8c882c6b84e83ae77895ad31abf2a_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111100d8c882c6b84e83ae77895ad31abf2a_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/bd0605a37ce144afb72d378d5f2a7772, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:21,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/bd0605a37ce144afb72d378d5f2a7772 is 175, key is test_row_0/A:col10/1731292280085/Put/seqid=0 2024-11-11T02:31:21,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741944_1120 (size=30955) 2024-11-11T02:31:21,051 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/bd0605a37ce144afb72d378d5f2a7772 2024-11-11T02:31:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T02:31:21,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/49d73717bf0d4717969ab51f2fb8967c is 50, key is test_row_0/B:col10/1731292280085/Put/seqid=0 2024-11-11T02:31:21,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741945_1121 (size=12001) 2024-11-11T02:31:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:21,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:21,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292341298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292341298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292341298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292341299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292341301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292341416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292341416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292341416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292341417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292341416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,498 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/49d73717bf0d4717969ab51f2fb8967c 2024-11-11T02:31:21,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/3e00b9c1fc1b458f83aa4dc0687023ce is 50, key is test_row_0/C:col10/1731292280085/Put/seqid=0 2024-11-11T02:31:21,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741946_1122 (size=12001) 2024-11-11T02:31:21,521 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/3e00b9c1fc1b458f83aa4dc0687023ce 2024-11-11T02:31:21,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/bd0605a37ce144afb72d378d5f2a7772 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/bd0605a37ce144afb72d378d5f2a7772 2024-11-11T02:31:21,535 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/bd0605a37ce144afb72d378d5f2a7772, entries=150, sequenceid=49, filesize=30.2 K 2024-11-11T02:31:21,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/49d73717bf0d4717969ab51f2fb8967c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/49d73717bf0d4717969ab51f2fb8967c 2024-11-11T02:31:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,542 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/49d73717bf0d4717969ab51f2fb8967c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T02:31:21,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/3e00b9c1fc1b458f83aa4dc0687023ce as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/3e00b9c1fc1b458f83aa4dc0687023ce 2024-11-11T02:31:21,549 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/3e00b9c1fc1b458f83aa4dc0687023ce, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T02:31:21,550 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for e33fc31e962e33bc26f57b81ef4a0f99 in 545ms, sequenceid=49, compaction requested=false 2024-11-11T02:31:21,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:21,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:21,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-11T02:31:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-11T02:31:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-11T02:31:21,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6270 sec 2024-11-11T02:31:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.6320 sec 2024-11-11T02:31:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,603 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T02:31:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:21,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T02:31:21,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:21,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:21,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:21,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b2e837c221b646a6afd8b0f8a5a56c69_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292281652/Put/seqid=0 2024-11-11T02:31:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741947_1123 (size=12154) 2024-11-11T02:31:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292341688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:21,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292341691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292341692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292341692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292341693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292341817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292341817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292341817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292341817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292341817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292342025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292342026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292342026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292342026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292342026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T02:31:22,066 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:22,067 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-11T02:31:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T02:31:22,071 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:22,072 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:22,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:22,092 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:22,097 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b2e837c221b646a6afd8b0f8a5a56c69_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b2e837c221b646a6afd8b0f8a5a56c69_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:22,098 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/cf92d79bd2c04dabbd08a82c8913c80b, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:22,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/cf92d79bd2c04dabbd08a82c8913c80b is 175, key is test_row_0/A:col10/1731292281652/Put/seqid=0 2024-11-11T02:31:22,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741948_1124 (size=30951) 2024-11-11T02:31:22,107 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/cf92d79bd2c04dabbd08a82c8913c80b 2024-11-11T02:31:22,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/4c1bf6b2c5514a56acf928573c59ad9c is 50, key is test_row_0/B:col10/1731292281652/Put/seqid=0 2024-11-11T02:31:22,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741949_1125 (size=9657) 2024-11-11T02:31:22,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/4c1bf6b2c5514a56acf928573c59ad9c 2024-11-11T02:31:22,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/c593a718b84a4a0c8e9dba1365b272eb is 50, key is test_row_0/C:col10/1731292281652/Put/seqid=0 2024-11-11T02:31:22,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741950_1126 (size=9657) 2024-11-11T02:31:22,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/c593a718b84a4a0c8e9dba1365b272eb 2024-11-11T02:31:22,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/cf92d79bd2c04dabbd08a82c8913c80b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/cf92d79bd2c04dabbd08a82c8913c80b 2024-11-11T02:31:22,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/cf92d79bd2c04dabbd08a82c8913c80b, entries=150, sequenceid=61, filesize=30.2 K 2024-11-11T02:31:22,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/4c1bf6b2c5514a56acf928573c59ad9c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/4c1bf6b2c5514a56acf928573c59ad9c 2024-11-11T02:31:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T02:31:22,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/4c1bf6b2c5514a56acf928573c59ad9c, entries=100, sequenceid=61, filesize=9.4 K 2024-11-11T02:31:22,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/c593a718b84a4a0c8e9dba1365b272eb as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/c593a718b84a4a0c8e9dba1365b272eb 2024-11-11T02:31:22,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/c593a718b84a4a0c8e9dba1365b272eb, entries=100, sequenceid=61, filesize=9.4 K 2024-11-11T02:31:22,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e33fc31e962e33bc26f57b81ef4a0f99 in 541ms, sequenceid=61, compaction requested=true 2024-11-11T02:31:22,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:22,195 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:22,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:22,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:22,196 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:22,197 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:22,197 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/A is initiating minor compaction (all files) 2024-11-11T02:31:22,197 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/A in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:22,197 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/d85fe9679127423c82bc55420716f7d4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/bd0605a37ce144afb72d378d5f2a7772, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/cf92d79bd2c04dabbd08a82c8913c80b] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=90.7 K 2024-11-11T02:31:22,197 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:22,197 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/d85fe9679127423c82bc55420716f7d4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/bd0605a37ce144afb72d378d5f2a7772, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/cf92d79bd2c04dabbd08a82c8913c80b] 2024-11-11T02:31:22,198 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting d85fe9679127423c82bc55420716f7d4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731292279942 2024-11-11T02:31:22,198 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:22,199 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/B is initiating minor compaction (all files) 2024-11-11T02:31:22,199 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/B in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:22,199 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/28fbcd689a3f43b7a110673ea3093d82, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/49d73717bf0d4717969ab51f2fb8967c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/4c1bf6b2c5514a56acf928573c59ad9c] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=32.9 K 2024-11-11T02:31:22,199 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd0605a37ce144afb72d378d5f2a7772, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731292280085 2024-11-11T02:31:22,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:22,199 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf92d79bd2c04dabbd08a82c8913c80b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731292281268 2024-11-11T02:31:22,199 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 28fbcd689a3f43b7a110673ea3093d82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731292279942 2024-11-11T02:31:22,200 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 49d73717bf0d4717969ab51f2fb8967c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731292280085 2024-11-11T02:31:22,200 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c1bf6b2c5514a56acf928573c59ad9c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731292281268 2024-11-11T02:31:22,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:22,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:22,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:22,213 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#B#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:22,214 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/0477c7c3d001437686dca46c9f7d8d61 is 50, key is test_row_0/B:col10/1731292281652/Put/seqid=0 2024-11-11T02:31:22,216 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:22,222 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111538922cc56b64fe7b6d65bcebb39f464_e33fc31e962e33bc26f57b81ef4a0f99 store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:22,225 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111538922cc56b64fe7b6d65bcebb39f464_e33fc31e962e33bc26f57b81ef4a0f99, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:22,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T02:31:22,225 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111538922cc56b64fe7b6d65bcebb39f464_e33fc31e962e33bc26f57b81ef4a0f99 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:22,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:22,225 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T02:31:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:22,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741951_1127 (size=12104) 2024-11-11T02:31:22,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cfb9430d530c4b84897bd8fbfa0bc06c_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292281692/Put/seqid=0 2024-11-11T02:31:22,252 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/0477c7c3d001437686dca46c9f7d8d61 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/0477c7c3d001437686dca46c9f7d8d61 2024-11-11T02:31:22,260 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/B of e33fc31e962e33bc26f57b81ef4a0f99 into 0477c7c3d001437686dca46c9f7d8d61(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:22,261 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:22,261 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/B, priority=13, startTime=1731292282196; duration=0sec 2024-11-11T02:31:22,261 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:22,261 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:B 2024-11-11T02:31:22,261 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:22,262 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:22,262 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/C is initiating minor compaction (all files) 2024-11-11T02:31:22,262 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/C in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:22,262 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/114095b410394c2ab2a4fa3fef7c6f4d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/3e00b9c1fc1b458f83aa4dc0687023ce, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/c593a718b84a4a0c8e9dba1365b272eb] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=32.9 K 2024-11-11T02:31:22,263 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 114095b410394c2ab2a4fa3fef7c6f4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731292279942 2024-11-11T02:31:22,263 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e00b9c1fc1b458f83aa4dc0687023ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731292280085 2024-11-11T02:31:22,263 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting c593a718b84a4a0c8e9dba1365b272eb, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731292281268 2024-11-11T02:31:22,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741952_1128 (size=4469) 2024-11-11T02:31:22,271 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#A#compaction#109 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:22,272 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/141ce89107304c329ca3fa47760320fa is 175, key is test_row_0/A:col10/1731292281652/Put/seqid=0 2024-11-11T02:31:22,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741953_1129 (size=12154) 2024-11-11T02:31:22,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:22,292 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#C#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:22,292 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/190b8ba269544aa4ba54bc86857401f0 is 50, key is test_row_0/C:col10/1731292281652/Put/seqid=0 2024-11-11T02:31:22,300 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cfb9430d530c4b84897bd8fbfa0bc06c_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cfb9430d530c4b84897bd8fbfa0bc06c_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:22,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/ab973ef1dea8426da11ceb15cdf9183b, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:22,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/ab973ef1dea8426da11ceb15cdf9183b is 175, key is test_row_0/A:col10/1731292281692/Put/seqid=0 2024-11-11T02:31:22,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741954_1130 (size=31165) 2024-11-11T02:31:22,318 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/141ce89107304c329ca3fa47760320fa as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/141ce89107304c329ca3fa47760320fa 2024-11-11T02:31:22,333 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/A of e33fc31e962e33bc26f57b81ef4a0f99 into 141ce89107304c329ca3fa47760320fa(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:22,333 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:22,333 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/A, priority=13, startTime=1731292282195; duration=0sec 2024-11-11T02:31:22,333 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:22,333 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:A 2024-11-11T02:31:22,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741955_1131 (size=12104) 2024-11-11T02:31:22,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741956_1132 (size=30955) 2024-11-11T02:31:22,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:22,345 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/ab973ef1dea8426da11ceb15cdf9183b 2024-11-11T02:31:22,347 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/190b8ba269544aa4ba54bc86857401f0 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/190b8ba269544aa4ba54bc86857401f0 2024-11-11T02:31:22,358 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/C of e33fc31e962e33bc26f57b81ef4a0f99 into 190b8ba269544aa4ba54bc86857401f0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/572e04d88de24381945dfe596270a536 is 50, key is test_row_0/B:col10/1731292281692/Put/seqid=0 2024-11-11T02:31:22,358 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:22,358 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/C, priority=13, startTime=1731292282200; duration=0sec 2024-11-11T02:31:22,358 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:22,358 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:C 2024-11-11T02:31:22,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292342356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292342359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292342360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292342359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292342361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741957_1133 (size=12001) 2024-11-11T02:31:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T02:31:22,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292342465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292342476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292342476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292342476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292342477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292342675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292342686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292342686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292342686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:22,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292342686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:22,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T02:31:22,775 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/572e04d88de24381945dfe596270a536 2024-11-11T02:31:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/fcf5a484656548e4919d328d1cfda54f is 50, key is test_row_0/C:col10/1731292281692/Put/seqid=0 2024-11-11T02:31:22,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741958_1134 (size=12001) 2024-11-11T02:31:22,790 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/fcf5a484656548e4919d328d1cfda54f 2024-11-11T02:31:22,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/ab973ef1dea8426da11ceb15cdf9183b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ab973ef1dea8426da11ceb15cdf9183b 2024-11-11T02:31:22,804 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ab973ef1dea8426da11ceb15cdf9183b, entries=150, sequenceid=81, filesize=30.2 K 2024-11-11T02:31:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/572e04d88de24381945dfe596270a536 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/572e04d88de24381945dfe596270a536 2024-11-11T02:31:22,813 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/572e04d88de24381945dfe596270a536, entries=150, sequenceid=81, filesize=11.7 K 2024-11-11T02:31:22,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/fcf5a484656548e4919d328d1cfda54f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/fcf5a484656548e4919d328d1cfda54f 2024-11-11T02:31:22,824 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/fcf5a484656548e4919d328d1cfda54f, entries=150, sequenceid=81, filesize=11.7 K 2024-11-11T02:31:22,825 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for e33fc31e962e33bc26f57b81ef4a0f99 in 600ms, sequenceid=81, compaction requested=false 2024-11-11T02:31:22,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:22,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:22,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-11T02:31:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-11T02:31:22,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-11T02:31:22,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 755 msec 2024-11-11T02:31:22,831 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 763 msec 2024-11-11T02:31:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:22,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T02:31:22,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:22,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:22,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:22,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:22,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:22,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:23,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111173f6e914662c487e9383a58b1307210f_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292282355/Put/seqid=0 2024-11-11T02:31:23,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741959_1135 (size=12154) 2024-11-11T02:31:23,019 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:23,027 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111173f6e914662c487e9383a58b1307210f_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111173f6e914662c487e9383a58b1307210f_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:23,030 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/9ae47f5978534435bfa813eb2b496479, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:23,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/9ae47f5978534435bfa813eb2b496479 is 175, key is test_row_0/A:col10/1731292282355/Put/seqid=0 2024-11-11T02:31:23,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292343016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292343031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292343032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292343034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292343035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741960_1136 (size=30955) 2024-11-11T02:31:23,043 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/9ae47f5978534435bfa813eb2b496479 2024-11-11T02:31:23,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/f2c0ad023be34275ad8530de9fb2c554 is 50, key is test_row_0/B:col10/1731292282355/Put/seqid=0 2024-11-11T02:31:23,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741961_1137 (size=12001) 2024-11-11T02:31:23,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292343135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292343145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292343146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292343146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292343146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T02:31:23,206 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:23,206 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-11T02:31:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T02:31:23,208 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:23,209 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:23,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T02:31:23,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292343346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T02:31:23,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:23,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:23,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:23,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292343355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:23,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292343355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292343355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292343356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/f2c0ad023be34275ad8530de9fb2c554 2024-11-11T02:31:23,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/fbe998d333bc4ce1babc9c788befe256 is 50, key is test_row_0/C:col10/1731292282355/Put/seqid=0 2024-11-11T02:31:23,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741962_1138 (size=12001) 2024-11-11T02:31:23,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T02:31:23,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:23,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:23,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:23,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:23,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T02:31:23,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292343656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292343665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T02:31:23,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:23,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:23,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:23,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:23,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:23,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292343675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292343675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292343676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:23,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T02:31:23,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:23,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:23,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:23,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:23,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T02:31:23,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/fbe998d333bc4ce1babc9c788befe256 2024-11-11T02:31:23,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/9ae47f5978534435bfa813eb2b496479 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9ae47f5978534435bfa813eb2b496479 2024-11-11T02:31:23,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9ae47f5978534435bfa813eb2b496479, entries=150, sequenceid=98, filesize=30.2 K 2024-11-11T02:31:23,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/f2c0ad023be34275ad8530de9fb2c554 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f2c0ad023be34275ad8530de9fb2c554 2024-11-11T02:31:23,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f2c0ad023be34275ad8530de9fb2c554, entries=150, sequenceid=98, filesize=11.7 K 2024-11-11T02:31:23,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/fbe998d333bc4ce1babc9c788befe256 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/fbe998d333bc4ce1babc9c788befe256 2024-11-11T02:31:23,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/fbe998d333bc4ce1babc9c788befe256, entries=150, sequenceid=98, filesize=11.7 K 2024-11-11T02:31:23,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for e33fc31e962e33bc26f57b81ef4a0f99 in 951ms, sequenceid=98, compaction requested=true 2024-11-11T02:31:23,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:23,941 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:23,942 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93075 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:23,943 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/A is initiating minor compaction (all files) 2024-11-11T02:31:23,943 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/A in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:23,943 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/141ce89107304c329ca3fa47760320fa, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ab973ef1dea8426da11ceb15cdf9183b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9ae47f5978534435bfa813eb2b496479] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=90.9 K 2024-11-11T02:31:23,943 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:23,943 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/141ce89107304c329ca3fa47760320fa, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ab973ef1dea8426da11ceb15cdf9183b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9ae47f5978534435bfa813eb2b496479] 2024-11-11T02:31:23,944 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 141ce89107304c329ca3fa47760320fa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731292280087 2024-11-11T02:31:23,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:23,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:23,944 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:23,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:23,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:23,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:23,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:23,945 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab973ef1dea8426da11ceb15cdf9183b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731292281663 2024-11-11T02:31:23,946 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ae47f5978534435bfa813eb2b496479, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731292282355 2024-11-11T02:31:23,946 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:23,946 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/B is initiating minor compaction (all files) 2024-11-11T02:31:23,947 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/B in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:23,947 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/0477c7c3d001437686dca46c9f7d8d61, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/572e04d88de24381945dfe596270a536, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f2c0ad023be34275ad8530de9fb2c554] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=35.3 K 2024-11-11T02:31:23,949 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 0477c7c3d001437686dca46c9f7d8d61, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731292280087 2024-11-11T02:31:23,950 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 572e04d88de24381945dfe596270a536, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731292281663 2024-11-11T02:31:23,951 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c0ad023be34275ad8530de9fb2c554, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731292282355 2024-11-11T02:31:23,962 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:23,967 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#B#compaction#118 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:23,968 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/b2f87cc3fa824a5c9b9534af6594c5cc is 50, key is test_row_0/B:col10/1731292282355/Put/seqid=0 2024-11-11T02:31:23,972 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411115d237ee9ad424a8c8d677945a7ea4bcf_e33fc31e962e33bc26f57b81ef4a0f99 store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:23,975 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411115d237ee9ad424a8c8d677945a7ea4bcf_e33fc31e962e33bc26f57b81ef4a0f99, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:23,975 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115d237ee9ad424a8c8d677945a7ea4bcf_e33fc31e962e33bc26f57b81ef4a0f99 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:23,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T02:31:23,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:23,977 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T02:31:23,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:23,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:23,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:23,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:23,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:23,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:23,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741963_1139 (size=12207) 2024-11-11T02:31:24,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741964_1140 (size=4469) 2024-11-11T02:31:24,008 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#A#compaction#117 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:24,009 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/71db1fcc23cf46bdab491d4c1fd53665 is 175, key is test_row_0/A:col10/1731292282355/Put/seqid=0 2024-11-11T02:31:24,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d70b1629518b49f0ae5a777b937eff99_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292283013/Put/seqid=0 2024-11-11T02:31:24,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741965_1141 (size=31161) 2024-11-11T02:31:24,025 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/71db1fcc23cf46bdab491d4c1fd53665 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/71db1fcc23cf46bdab491d4c1fd53665 2024-11-11T02:31:24,032 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/A of e33fc31e962e33bc26f57b81ef4a0f99 into 71db1fcc23cf46bdab491d4c1fd53665(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:24,032 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:24,032 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/A, priority=13, startTime=1731292283941; duration=0sec 2024-11-11T02:31:24,032 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:24,032 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:A 2024-11-11T02:31:24,032 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:24,035 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:24,035 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/C is initiating minor compaction (all files) 2024-11-11T02:31:24,035 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/C in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:24,035 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/190b8ba269544aa4ba54bc86857401f0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/fcf5a484656548e4919d328d1cfda54f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/fbe998d333bc4ce1babc9c788befe256] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=35.3 K 2024-11-11T02:31:24,036 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 190b8ba269544aa4ba54bc86857401f0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731292280087 2024-11-11T02:31:24,036 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf5a484656548e4919d328d1cfda54f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731292281663 2024-11-11T02:31:24,037 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbe998d333bc4ce1babc9c788befe256, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731292282355 2024-11-11T02:31:24,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741966_1142 (size=12154) 2024-11-11T02:31:24,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:24,054 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d70b1629518b49f0ae5a777b937eff99_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d70b1629518b49f0ae5a777b937eff99_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:24,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/344626f82d8347b3952ecc80fac626fc, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:24,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/344626f82d8347b3952ecc80fac626fc is 175, key is test_row_0/A:col10/1731292283013/Put/seqid=0 2024-11-11T02:31:24,058 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#C#compaction#120 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:24,059 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/cf0bf35df65f4d77a83d1da128136d6b is 50, key is test_row_0/C:col10/1731292282355/Put/seqid=0 2024-11-11T02:31:24,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741967_1143 (size=30955) 2024-11-11T02:31:24,079 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/344626f82d8347b3952ecc80fac626fc 2024-11-11T02:31:24,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741968_1144 (size=12207) 2024-11-11T02:31:24,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/f93af1f14ec84248ab453de3dbb310d0 is 50, key is test_row_0/B:col10/1731292283013/Put/seqid=0 2024-11-11T02:31:24,097 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/cf0bf35df65f4d77a83d1da128136d6b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cf0bf35df65f4d77a83d1da128136d6b 2024-11-11T02:31:24,108 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/C of e33fc31e962e33bc26f57b81ef4a0f99 into cf0bf35df65f4d77a83d1da128136d6b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:24,108 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:24,108 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/C, priority=13, startTime=1731292283944; duration=0sec 2024-11-11T02:31:24,109 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:24,109 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:C 2024-11-11T02:31:24,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741969_1145 (size=12001) 2024-11-11T02:31:24,114 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/f93af1f14ec84248ab453de3dbb310d0 2024-11-11T02:31:24,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/d3861670e01448bd875d5e8db6aac787 is 50, key is test_row_0/C:col10/1731292283013/Put/seqid=0 2024-11-11T02:31:24,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741970_1146 (size=12001) 2024-11-11T02:31:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:24,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:24,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292344185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:24,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:24,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292344186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292344186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:24,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292344189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292344190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:24,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292344306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292344306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T02:31:24,401 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/b2f87cc3fa824a5c9b9534af6594c5cc as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b2f87cc3fa824a5c9b9534af6594c5cc 2024-11-11T02:31:24,408 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/B of e33fc31e962e33bc26f57b81ef4a0f99 into b2f87cc3fa824a5c9b9534af6594c5cc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:24,409 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:24,409 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/B, priority=13, startTime=1731292283944; duration=0sec 2024-11-11T02:31:24,409 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:24,409 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:B 2024-11-11T02:31:24,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292344516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292344516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:24,529 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/d3861670e01448bd875d5e8db6aac787 2024-11-11T02:31:24,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/344626f82d8347b3952ecc80fac626fc as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/344626f82d8347b3952ecc80fac626fc 2024-11-11T02:31:24,543 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/344626f82d8347b3952ecc80fac626fc, entries=150, sequenceid=118, filesize=30.2 K 2024-11-11T02:31:24,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/f93af1f14ec84248ab453de3dbb310d0 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f93af1f14ec84248ab453de3dbb310d0 2024-11-11T02:31:24,551 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f93af1f14ec84248ab453de3dbb310d0, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T02:31:24,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/d3861670e01448bd875d5e8db6aac787 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/d3861670e01448bd875d5e8db6aac787 2024-11-11T02:31:24,562 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/d3861670e01448bd875d5e8db6aac787, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T02:31:24,564 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e33fc31e962e33bc26f57b81ef4a0f99 in 587ms, sequenceid=118, compaction requested=false 2024-11-11T02:31:24,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:24,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:24,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-11T02:31:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-11T02:31:24,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-11T02:31:24,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3570 sec 2024-11-11T02:31:24,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.3650 sec 2024-11-11T02:31:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:24,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T02:31:24,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:24,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:24,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:24,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:24,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:24,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:24,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111102f7a1ffeed9415fa100c4bc59abb74f_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292284827/Put/seqid=0 2024-11-11T02:31:24,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741971_1147 (size=12254) 2024-11-11T02:31:24,842 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:24,849 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111102f7a1ffeed9415fa100c4bc59abb74f_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111102f7a1ffeed9415fa100c4bc59abb74f_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:24,851 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/e73e539eadba47dda202182b403626b7, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:24,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/e73e539eadba47dda202182b403626b7 is 175, key is test_row_0/A:col10/1731292284827/Put/seqid=0 2024-11-11T02:31:24,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741972_1148 (size=31055) 2024-11-11T02:31:24,862 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/e73e539eadba47dda202182b403626b7 2024-11-11T02:31:24,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/d00b3721396b4ef6b247a434925587aa is 50, key is test_row_0/B:col10/1731292284827/Put/seqid=0 2024-11-11T02:31:24,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292344875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:24,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292344877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:24,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741973_1149 (size=12101) 2024-11-11T02:31:24,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/d00b3721396b4ef6b247a434925587aa 2024-11-11T02:31:24,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/0a9f033a3df244aeac256d33bb96f8e6 is 50, key is test_row_0/C:col10/1731292284827/Put/seqid=0 2024-11-11T02:31:24,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741974_1150 (size=12101) 2024-11-11T02:31:24,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/0a9f033a3df244aeac256d33bb96f8e6 2024-11-11T02:31:24,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/e73e539eadba47dda202182b403626b7 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/e73e539eadba47dda202182b403626b7 2024-11-11T02:31:24,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/e73e539eadba47dda202182b403626b7, entries=150, sequenceid=135, filesize=30.3 K 2024-11-11T02:31:24,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/d00b3721396b4ef6b247a434925587aa as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d00b3721396b4ef6b247a434925587aa 2024-11-11T02:31:24,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d00b3721396b4ef6b247a434925587aa, entries=150, sequenceid=135, filesize=11.8 K 2024-11-11T02:31:24,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/0a9f033a3df244aeac256d33bb96f8e6 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/0a9f033a3df244aeac256d33bb96f8e6 2024-11-11T02:31:24,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/0a9f033a3df244aeac256d33bb96f8e6, entries=150, sequenceid=135, filesize=11.8 K 2024-11-11T02:31:24,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e33fc31e962e33bc26f57b81ef4a0f99 in 133ms, sequenceid=135, compaction requested=true 2024-11-11T02:31:24,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:24,962 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:24,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:24,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:24,962 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:24,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:24,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:24,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:24,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:24,963 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:24,963 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/A is initiating minor compaction (all files) 2024-11-11T02:31:24,963 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/A in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:24,963 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:24,964 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/B is initiating minor compaction (all files) 2024-11-11T02:31:24,964 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/71db1fcc23cf46bdab491d4c1fd53665, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/344626f82d8347b3952ecc80fac626fc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/e73e539eadba47dda202182b403626b7] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=91.0 K 2024-11-11T02:31:24,964 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/B in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:24,964 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:24,964 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b2f87cc3fa824a5c9b9534af6594c5cc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f93af1f14ec84248ab453de3dbb310d0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d00b3721396b4ef6b247a434925587aa] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=35.5 K 2024-11-11T02:31:24,964 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/71db1fcc23cf46bdab491d4c1fd53665, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/344626f82d8347b3952ecc80fac626fc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/e73e539eadba47dda202182b403626b7] 2024-11-11T02:31:24,964 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71db1fcc23cf46bdab491d4c1fd53665, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731292282355 2024-11-11T02:31:24,964 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f87cc3fa824a5c9b9534af6594c5cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731292282355 2024-11-11T02:31:24,965 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting f93af1f14ec84248ab453de3dbb310d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292283013 2024-11-11T02:31:24,965 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 344626f82d8347b3952ecc80fac626fc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292283013 2024-11-11T02:31:24,966 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting d00b3721396b4ef6b247a434925587aa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731292284178 2024-11-11T02:31:24,966 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting e73e539eadba47dda202182b403626b7, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731292284178 2024-11-11T02:31:24,979 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:24,980 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/1959499c07734bc7b613ae569ccf903f is 50, key is test_row_0/B:col10/1731292284827/Put/seqid=0 2024-11-11T02:31:24,982 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:24,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T02:31:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:24,998 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111128431557740a461fa8a6dcc3867e07eb_e33fc31e962e33bc26f57b81ef4a0f99 store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:25,001 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111128431557740a461fa8a6dcc3867e07eb_e33fc31e962e33bc26f57b81ef4a0f99, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:25,001 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111128431557740a461fa8a6dcc3867e07eb_e33fc31e962e33bc26f57b81ef4a0f99 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:25,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741975_1151 (size=12409) 2024-11-11T02:31:25,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292345019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:25,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292345021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:25,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741976_1152 (size=4469) 2024-11-11T02:31:25,035 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#A#compaction#127 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:25,035 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/229672d84e2346eb8466ace05d3cc792 is 175, key is test_row_0/A:col10/1731292284827/Put/seqid=0 2024-11-11T02:31:25,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114fd598d1d6454740862abc1a0c3affdd_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292284987/Put/seqid=0 2024-11-11T02:31:25,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741977_1153 (size=31363) 2024-11-11T02:31:25,078 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/229672d84e2346eb8466ace05d3cc792 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/229672d84e2346eb8466ace05d3cc792 2024-11-11T02:31:25,086 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/A of e33fc31e962e33bc26f57b81ef4a0f99 into 229672d84e2346eb8466ace05d3cc792(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:25,086 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:25,086 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/A, priority=13, startTime=1731292284961; duration=0sec 2024-11-11T02:31:25,086 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:25,086 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:A 2024-11-11T02:31:25,086 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:25,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741978_1154 (size=14794) 2024-11-11T02:31:25,093 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:25,093 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/C is initiating minor compaction (all files) 2024-11-11T02:31:25,093 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/C in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:25,093 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cf0bf35df65f4d77a83d1da128136d6b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/d3861670e01448bd875d5e8db6aac787, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/0a9f033a3df244aeac256d33bb96f8e6] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=35.5 K 2024-11-11T02:31:25,094 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf0bf35df65f4d77a83d1da128136d6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731292282355 2024-11-11T02:31:25,094 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3861670e01448bd875d5e8db6aac787, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292283013 2024-11-11T02:31:25,094 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a9f033a3df244aeac256d33bb96f8e6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731292284178 2024-11-11T02:31:25,103 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#C#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:25,103 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/cc838f1e165442d1baaf332db8738a76 is 50, key is test_row_0/C:col10/1731292284827/Put/seqid=0 2024-11-11T02:31:25,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741979_1155 (size=12409) 2024-11-11T02:31:25,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292345136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:25,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292345136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:25,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292345206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:25,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:25,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292345206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292345206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:25,211 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in e33fc31e962e33bc26f57b81ef4a0f99 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:25,211 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in e33fc31e962e33bc26f57b81ef4a0f99 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:25,212 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in e33fc31e962e33bc26f57b81ef4a0f99 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:25,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292345345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:25,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292345345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:25,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T02:31:25,356 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:25,356 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-11T02:31:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T02:31:25,358 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:25,359 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:25,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:25,430 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/1959499c07734bc7b613ae569ccf903f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/1959499c07734bc7b613ae569ccf903f 2024-11-11T02:31:25,437 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/B of e33fc31e962e33bc26f57b81ef4a0f99 into 1959499c07734bc7b613ae569ccf903f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:25,437 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:25,437 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/B, priority=13, startTime=1731292284962; duration=0sec 2024-11-11T02:31:25,437 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:25,437 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:B 2024-11-11T02:31:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T02:31:25,493 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:25,497 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114fd598d1d6454740862abc1a0c3affdd_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114fd598d1d6454740862abc1a0c3affdd_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:25,497 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/4f5c2ae4d2ee40bb9bf9e42f14cc24ca, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:25,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/4f5c2ae4d2ee40bb9bf9e42f14cc24ca is 175, key is test_row_0/A:col10/1731292284987/Put/seqid=0 2024-11-11T02:31:25,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741980_1156 (size=39749) 2024-11-11T02:31:25,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T02:31:25,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:25,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:25,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:25,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:25,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:25,519 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/cc838f1e165442d1baaf332db8738a76 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cc838f1e165442d1baaf332db8738a76 2024-11-11T02:31:25,524 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/C of e33fc31e962e33bc26f57b81ef4a0f99 into cc838f1e165442d1baaf332db8738a76(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:25,524 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:25,524 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/C, priority=13, startTime=1731292284962; duration=0sec 2024-11-11T02:31:25,524 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:25,525 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:C 2024-11-11T02:31:25,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292345656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:25,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292345656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:25,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T02:31:25,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:25,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:25,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:25,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:25,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T02:31:25,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T02:31:25,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:25,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:25,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:25,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:25,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:25,902 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/4f5c2ae4d2ee40bb9bf9e42f14cc24ca 2024-11-11T02:31:25,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/c8b4de2ccfe542e7a3fbdcda36a02d4e is 50, key is test_row_0/B:col10/1731292284987/Put/seqid=0 2024-11-11T02:31:25,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741981_1157 (size=12151) 2024-11-11T02:31:25,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/c8b4de2ccfe542e7a3fbdcda36a02d4e 2024-11-11T02:31:25,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/787882435a884ca49ca49f3c54a1a348 is 50, key is test_row_0/C:col10/1731292284987/Put/seqid=0 2024-11-11T02:31:25,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741982_1158 (size=12151) 2024-11-11T02:31:25,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T02:31:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:25,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T02:31:26,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T02:31:26,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:26,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:26,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:26,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:26,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292346166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:26,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292346166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:26,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T02:31:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:26,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:26,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/787882435a884ca49ca49f3c54a1a348 2024-11-11T02:31:26,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/4f5c2ae4d2ee40bb9bf9e42f14cc24ca as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/4f5c2ae4d2ee40bb9bf9e42f14cc24ca 2024-11-11T02:31:26,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/4f5c2ae4d2ee40bb9bf9e42f14cc24ca, entries=200, sequenceid=155, filesize=38.8 K 2024-11-11T02:31:26,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/c8b4de2ccfe542e7a3fbdcda36a02d4e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/c8b4de2ccfe542e7a3fbdcda36a02d4e 2024-11-11T02:31:26,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/c8b4de2ccfe542e7a3fbdcda36a02d4e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T02:31:26,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/787882435a884ca49ca49f3c54a1a348 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/787882435a884ca49ca49f3c54a1a348 2024-11-11T02:31:26,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/787882435a884ca49ca49f3c54a1a348, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T02:31:26,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e33fc31e962e33bc26f57b81ef4a0f99 in 1365ms, sequenceid=155, compaction requested=false 2024-11-11T02:31:26,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:26,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T02:31:26,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:26,430 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T02:31:26,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:26,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:26,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:26,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:26,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:26,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:26,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114ad67d2314df4168932ee9c123542d8e_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292285011/Put/seqid=0 2024-11-11T02:31:26,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741983_1159 (size=12304) 2024-11-11T02:31:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T02:31:26,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:26,851 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114ad67d2314df4168932ee9c123542d8e_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114ad67d2314df4168932ee9c123542d8e_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:26,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/ca1ee4764efd48daa54a1dfd150da053, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:26,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/ca1ee4764efd48daa54a1dfd150da053 is 175, key is test_row_0/A:col10/1731292285011/Put/seqid=0 2024-11-11T02:31:26,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741984_1160 (size=31105) 2024-11-11T02:31:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:27,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:27,210 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T02:31:27,210 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T02:31:27,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292347236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292347237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292347239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731292287229, max timestamp=1731292287239Num uniques 859; 2024-11-11T02:31:27,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T02:31:27,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731292287229, max timestamp=1731292287239Num uniques 859; 2024-11-11T02:31:27,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731292287229, max timestamp=1731292287239Num uniques 859; 2024-11-11T02:31:27,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T02:31:27,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T02:31:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,258 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/ca1ee4764efd48daa54a1dfd150da053 2024-11-11T02:31:27,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/8fb9473a6f774b65aa9cb8cd198b4202 is 50, key is test_row_0/B:col10/1731292285011/Put/seqid=0 2024-11-11T02:31:27,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741985_1161 (size=12151) 2024-11-11T02:31:27,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292347281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292347283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292347345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292347346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292347356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292347395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292347405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T02:31:27,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292347556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292347556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292347565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292347606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292347616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,672 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/8fb9473a6f774b65aa9cb8cd198b4202 2024-11-11T02:31:27,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/ccebabd1143e4cc8b1fa25f9f16bb15a is 50, key is test_row_0/C:col10/1731292285011/Put/seqid=0 2024-11-11T02:31:27,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741986_1162 (size=12151) 2024-11-11T02:31:27,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292347866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292347866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292347875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292347915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:27,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292347926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:28,082 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/ccebabd1143e4cc8b1fa25f9f16bb15a 2024-11-11T02:31:28,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/ca1ee4764efd48daa54a1dfd150da053 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ca1ee4764efd48daa54a1dfd150da053 2024-11-11T02:31:28,091 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ca1ee4764efd48daa54a1dfd150da053, entries=150, sequenceid=170, filesize=30.4 K 2024-11-11T02:31:28,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/8fb9473a6f774b65aa9cb8cd198b4202 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/8fb9473a6f774b65aa9cb8cd198b4202 2024-11-11T02:31:28,095 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/8fb9473a6f774b65aa9cb8cd198b4202, entries=150, sequenceid=170, filesize=11.9 K 2024-11-11T02:31:28,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/ccebabd1143e4cc8b1fa25f9f16bb15a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/ccebabd1143e4cc8b1fa25f9f16bb15a 2024-11-11T02:31:28,100 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/ccebabd1143e4cc8b1fa25f9f16bb15a, entries=150, sequenceid=170, filesize=11.9 K 2024-11-11T02:31:28,101 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for e33fc31e962e33bc26f57b81ef4a0f99 in 1671ms, sequenceid=170, compaction requested=true 2024-11-11T02:31:28,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:28,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:28,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-11T02:31:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-11T02:31:28,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-11T02:31:28,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7430 sec 2024-11-11T02:31:28,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.7480 sec 2024-11-11T02:31:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:28,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-11T02:31:28,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:28,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:28,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:28,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:28,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:28,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:28,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111737e338727bf4b2c91973ee800609acf_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292287279/Put/seqid=0 2024-11-11T02:31:28,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741987_1163 (size=14794) 2024-11-11T02:31:28,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292348400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:28,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292348401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292348402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:28,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292348426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:28,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292348436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:28,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292348515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:28,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292348516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292348516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:28,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292348725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:28,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292348725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292348726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:28,796 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:28,799 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111737e338727bf4b2c91973ee800609acf_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111737e338727bf4b2c91973ee800609acf_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:28,800 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/5696ab62539e484da59ee91a0bbcc575, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:28,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/5696ab62539e484da59ee91a0bbcc575 is 175, key is test_row_0/A:col10/1731292287279/Put/seqid=0 2024-11-11T02:31:28,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741988_1164 (size=39749) 2024-11-11T02:31:29,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292349036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:29,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292349036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292349036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:29,205 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/5696ab62539e484da59ee91a0bbcc575 2024-11-11T02:31:29,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/b78a867e876d4fd3914e3af416ea161f is 50, key is test_row_0/B:col10/1731292287279/Put/seqid=0 2024-11-11T02:31:29,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741989_1165 (size=12151) 2024-11-11T02:31:29,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/b78a867e876d4fd3914e3af416ea161f 2024-11-11T02:31:29,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/bdfe57ecd09f443b8a7f3a2c832e5e03 is 50, key is test_row_0/C:col10/1731292287279/Put/seqid=0 2024-11-11T02:31:29,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741990_1166 (size=12151) 2024-11-11T02:31:29,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292349445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:29,447 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in e33fc31e962e33bc26f57b81ef4a0f99 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:29,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292349455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:29,461 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in e33fc31e962e33bc26f57b81ef4a0f99 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T02:31:29,525 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:29,526 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-11T02:31:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T02:31:29,527 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:29,528 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:29,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:29,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292349545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:29,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292349555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292349555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:29,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/bdfe57ecd09f443b8a7f3a2c832e5e03 2024-11-11T02:31:29,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/5696ab62539e484da59ee91a0bbcc575 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/5696ab62539e484da59ee91a0bbcc575 2024-11-11T02:31:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T02:31:29,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/5696ab62539e484da59ee91a0bbcc575, entries=200, sequenceid=204, filesize=38.8 K 2024-11-11T02:31:29,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/b78a867e876d4fd3914e3af416ea161f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b78a867e876d4fd3914e3af416ea161f 2024-11-11T02:31:29,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b78a867e876d4fd3914e3af416ea161f, entries=150, sequenceid=204, filesize=11.9 K 2024-11-11T02:31:29,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/bdfe57ecd09f443b8a7f3a2c832e5e03 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/bdfe57ecd09f443b8a7f3a2c832e5e03 2024-11-11T02:31:29,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/bdfe57ecd09f443b8a7f3a2c832e5e03, entries=150, sequenceid=204, filesize=11.9 K 2024-11-11T02:31:29,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for e33fc31e962e33bc26f57b81ef4a0f99 in 1272ms, sequenceid=204, compaction requested=true 2024-11-11T02:31:29,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:29,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:29,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:29,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:29,650 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:29,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:29,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:29,650 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:29,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:29,651 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141966 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:29,651 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:29,651 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/A is initiating minor compaction (all files) 2024-11-11T02:31:29,651 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/B is initiating minor compaction (all files) 2024-11-11T02:31:29,651 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/A in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:29,651 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/B in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:29,651 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/1959499c07734bc7b613ae569ccf903f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/c8b4de2ccfe542e7a3fbdcda36a02d4e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/8fb9473a6f774b65aa9cb8cd198b4202, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b78a867e876d4fd3914e3af416ea161f] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=47.7 K 2024-11-11T02:31:29,651 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/229672d84e2346eb8466ace05d3cc792, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/4f5c2ae4d2ee40bb9bf9e42f14cc24ca, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ca1ee4764efd48daa54a1dfd150da053, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/5696ab62539e484da59ee91a0bbcc575] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=138.6 K 2024-11-11T02:31:29,651 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:29,651 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/229672d84e2346eb8466ace05d3cc792, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/4f5c2ae4d2ee40bb9bf9e42f14cc24ca, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ca1ee4764efd48daa54a1dfd150da053, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/5696ab62539e484da59ee91a0bbcc575] 2024-11-11T02:31:29,652 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 1959499c07734bc7b613ae569ccf903f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731292284178 2024-11-11T02:31:29,652 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 229672d84e2346eb8466ace05d3cc792, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731292284178 2024-11-11T02:31:29,652 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting c8b4de2ccfe542e7a3fbdcda36a02d4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731292284866 2024-11-11T02:31:29,652 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f5c2ae4d2ee40bb9bf9e42f14cc24ca, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731292284866 2024-11-11T02:31:29,652 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca1ee4764efd48daa54a1dfd150da053, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731292285003 2024-11-11T02:31:29,652 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fb9473a6f774b65aa9cb8cd198b4202, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731292285003 2024-11-11T02:31:29,653 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5696ab62539e484da59ee91a0bbcc575, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292287271 2024-11-11T02:31:29,653 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting b78a867e876d4fd3914e3af416ea161f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292287279 2024-11-11T02:31:29,659 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:29,661 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:29,661 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/4277867e578c4f72b1d8c276fa5d6898 is 50, key is test_row_0/B:col10/1731292287279/Put/seqid=0 2024-11-11T02:31:29,663 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111a88a079f07034389b4982ee40ed95b40_e33fc31e962e33bc26f57b81ef4a0f99 store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:29,665 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111a88a079f07034389b4982ee40ed95b40_e33fc31e962e33bc26f57b81ef4a0f99, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:29,665 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a88a079f07034389b4982ee40ed95b40_e33fc31e962e33bc26f57b81ef4a0f99 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:29,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741991_1167 (size=12595) 2024-11-11T02:31:29,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T02:31:29,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:29,680 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T02:31:29,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:29,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:29,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:29,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:29,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:29,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:29,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741992_1168 (size=4469) 2024-11-11T02:31:29,690 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#A#compaction#139 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:29,691 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/b7f0eefc3b6d40fc9dad50bcbf6d3828 is 175, key is test_row_0/A:col10/1731292287279/Put/seqid=0 2024-11-11T02:31:29,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741993_1169 (size=31549) 2024-11-11T02:31:29,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111137b2f3779ba74aed9b110c13c6f54ee8_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292288401/Put/seqid=0 2024-11-11T02:31:29,720 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/b7f0eefc3b6d40fc9dad50bcbf6d3828 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/b7f0eefc3b6d40fc9dad50bcbf6d3828 2024-11-11T02:31:29,728 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/A of e33fc31e962e33bc26f57b81ef4a0f99 into b7f0eefc3b6d40fc9dad50bcbf6d3828(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:29,728 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:29,728 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/A, priority=12, startTime=1731292289650; duration=0sec 2024-11-11T02:31:29,728 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:29,728 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:A 2024-11-11T02:31:29,728 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:29,730 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:29,730 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/C is initiating minor compaction (all files) 2024-11-11T02:31:29,730 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/C in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:29,730 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cc838f1e165442d1baaf332db8738a76, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/787882435a884ca49ca49f3c54a1a348, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/ccebabd1143e4cc8b1fa25f9f16bb15a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/bdfe57ecd09f443b8a7f3a2c832e5e03] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=47.7 K 2024-11-11T02:31:29,731 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc838f1e165442d1baaf332db8738a76, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731292284178 2024-11-11T02:31:29,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741994_1170 (size=12304) 2024-11-11T02:31:29,732 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 787882435a884ca49ca49f3c54a1a348, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731292284866 2024-11-11T02:31:29,732 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccebabd1143e4cc8b1fa25f9f16bb15a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731292285003 2024-11-11T02:31:29,732 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdfe57ecd09f443b8a7f3a2c832e5e03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292287279 2024-11-11T02:31:29,747 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:29,748 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/1fa45d6fb3e440bb8ee52ca2ff0a2563 is 50, key is test_row_0/C:col10/1731292287279/Put/seqid=0 2024-11-11T02:31:29,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741995_1171 (size=12595) 2024-11-11T02:31:29,773 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/1fa45d6fb3e440bb8ee52ca2ff0a2563 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/1fa45d6fb3e440bb8ee52ca2ff0a2563 2024-11-11T02:31:29,785 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/C of e33fc31e962e33bc26f57b81ef4a0f99 into 1fa45d6fb3e440bb8ee52ca2ff0a2563(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:29,785 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:29,785 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/C, priority=12, startTime=1731292289650; duration=0sec 2024-11-11T02:31:29,785 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:29,785 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:C 2024-11-11T02:31:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T02:31:30,072 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/4277867e578c4f72b1d8c276fa5d6898 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/4277867e578c4f72b1d8c276fa5d6898 2024-11-11T02:31:30,077 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/B of e33fc31e962e33bc26f57b81ef4a0f99 into 4277867e578c4f72b1d8c276fa5d6898(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:30,077 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:30,077 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/B, priority=12, startTime=1731292289650; duration=0sec 2024-11-11T02:31:30,077 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:30,077 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:B 2024-11-11T02:31:30,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:30,137 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111137b2f3779ba74aed9b110c13c6f54ee8_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111137b2f3779ba74aed9b110c13c6f54ee8_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:30,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/df1b3aeae8e74d9f9c94b707c61a4ed9, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/df1b3aeae8e74d9f9c94b707c61a4ed9 is 175, key is test_row_0/A:col10/1731292288401/Put/seqid=0 2024-11-11T02:31:30,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741996_1172 (size=31105) 2024-11-11T02:31:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T02:31:30,545 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/df1b3aeae8e74d9f9c94b707c61a4ed9 2024-11-11T02:31:30,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/315d90da515a4ed1b3a435c5ae853310 is 50, key is test_row_0/B:col10/1731292288401/Put/seqid=0 2024-11-11T02:31:30,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741997_1173 (size=12151) 2024-11-11T02:31:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:30,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:30,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292290592, max timestamp=1731292290599Num uniques 859; 2024-11-11T02:31:30,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:31:30,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292290592, max timestamp=1731292290599Num uniques 859; 2024-11-11T02:31:30,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:31:30,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292290592, max timestamp=1731292290599Num uniques 859; 2024-11-11T02:31:30,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:31:30,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292350635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:30,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:30,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292350636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:30,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292350637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:30,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:30,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T02:31:30,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292350746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:30,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292350755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292350755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:30,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292350956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:30,959 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/315d90da515a4ed1b3a435c5ae853310 2024-11-11T02:31:30,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/2c841996fafb4181a91aad43263d0cad is 50, key is test_row_0/C:col10/1731292288401/Put/seqid=0 2024-11-11T02:31:30,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741998_1174 (size=12151) 2024-11-11T02:31:30,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292350965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:30,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292350965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:31,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292351265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:31,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292351276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292351276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:31,371 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/2c841996fafb4181a91aad43263d0cad 2024-11-11T02:31:31,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/df1b3aeae8e74d9f9c94b707c61a4ed9 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/df1b3aeae8e74d9f9c94b707c61a4ed9 2024-11-11T02:31:31,381 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/df1b3aeae8e74d9f9c94b707c61a4ed9, entries=150, sequenceid=215, filesize=30.4 K 2024-11-11T02:31:31,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/315d90da515a4ed1b3a435c5ae853310 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/315d90da515a4ed1b3a435c5ae853310 2024-11-11T02:31:31,387 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/315d90da515a4ed1b3a435c5ae853310, entries=150, sequenceid=215, filesize=11.9 K 2024-11-11T02:31:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/2c841996fafb4181a91aad43263d0cad as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/2c841996fafb4181a91aad43263d0cad 2024-11-11T02:31:31,393 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/2c841996fafb4181a91aad43263d0cad, entries=150, sequenceid=215, filesize=11.9 K 2024-11-11T02:31:31,394 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for e33fc31e962e33bc26f57b81ef4a0f99 in 1714ms, sequenceid=215, compaction requested=false 2024-11-11T02:31:31,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:31,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:31,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-11T02:31:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-11T02:31:31,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-11T02:31:31,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8680 sec 2024-11-11T02:31:31,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.8720 sec 2024-11-11T02:31:31,463 INFO [master/370bc2ade342:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T02:31:31,463 INFO [master/370bc2ade342:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T02:31:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:31,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-11T02:31:31,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:31,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:31:31,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:31,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:31:31,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:31,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:31:31,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111146b858ea7398454cb0e45b87b2a2ff1c_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292290635/Put/seqid=0 2024-11-11T02:31:31,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292351480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:31,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:31,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741999_1175 (size=12304) 2024-11-11T02:31:31,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292351486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:31,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292351595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:31,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292351596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T02:31:31,686 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:31,686 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-11T02:31:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T02:31:31,687 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:31,688 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:31,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:31,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292351775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:31,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292351785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:31,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292351785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T02:31:31,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292351806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:31,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292351806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:31,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T02:31:31,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:31,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:31,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:31,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:31,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:31,889 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:31,892 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111146b858ea7398454cb0e45b87b2a2ff1c_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146b858ea7398454cb0e45b87b2a2ff1c_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:31,893 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/5100a46f7d6742c3b5f94ad7c834f059, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:31,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/5100a46f7d6742c3b5f94ad7c834f059 is 175, key is test_row_0/A:col10/1731292290635/Put/seqid=0 2024-11-11T02:31:31,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742000_1176 (size=31105) 2024-11-11T02:31:31,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T02:31:31,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:31,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:31,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:31,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:31,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:31,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T02:31:32,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292352115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:32,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292352116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:32,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T02:31:32,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:32,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:32,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:32,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:32,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:32,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T02:31:32,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:32,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:32,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:32,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:32,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:32,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:32,300 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/5100a46f7d6742c3b5f94ad7c834f059 2024-11-11T02:31:32,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/d6e65565d357402b9f979c1053c0885b is 50, key is test_row_0/B:col10/1731292290635/Put/seqid=0 2024-11-11T02:31:32,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742001_1177 (size=12151) 2024-11-11T02:31:32,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/d6e65565d357402b9f979c1053c0885b 2024-11-11T02:31:32,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T02:31:32,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/497c01272d954b5caa59501a0970bd7a is 50, key is test_row_0/C:col10/1731292290635/Put/seqid=0 2024-11-11T02:31:32,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742002_1178 (size=12151) 2024-11-11T02:31:32,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T02:31:32,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:32,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:32,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:32,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:32,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:32,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T02:31:32,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:32,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:32,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:32,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:32,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:32,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292352625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:32,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292352626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:32,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/497c01272d954b5caa59501a0970bd7a 2024-11-11T02:31:32,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/5100a46f7d6742c3b5f94ad7c834f059 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/5100a46f7d6742c3b5f94ad7c834f059 2024-11-11T02:31:32,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/5100a46f7d6742c3b5f94ad7c834f059, entries=150, sequenceid=256, filesize=30.4 K 2024-11-11T02:31:32,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/d6e65565d357402b9f979c1053c0885b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d6e65565d357402b9f979c1053c0885b 2024-11-11T02:31:32,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d6e65565d357402b9f979c1053c0885b, entries=150, sequenceid=256, filesize=11.9 K 2024-11-11T02:31:32,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/497c01272d954b5caa59501a0970bd7a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/497c01272d954b5caa59501a0970bd7a 2024-11-11T02:31:32,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/497c01272d954b5caa59501a0970bd7a, entries=150, sequenceid=256, filesize=11.9 K 2024-11-11T02:31:32,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for e33fc31e962e33bc26f57b81ef4a0f99 in 1274ms, sequenceid=256, compaction requested=true 2024-11-11T02:31:32,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:32,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:32,748 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:32,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:32,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:32,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:32,748 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:32,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:32,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:32,750 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:32,750 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/B is initiating minor compaction (all files) 2024-11-11T02:31:32,750 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:32,750 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/A is initiating minor compaction (all files) 2024-11-11T02:31:32,750 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/B in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:32,750 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/A in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:32,750 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/b7f0eefc3b6d40fc9dad50bcbf6d3828, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/df1b3aeae8e74d9f9c94b707c61a4ed9, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/5100a46f7d6742c3b5f94ad7c834f059] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=91.6 K 2024-11-11T02:31:32,750 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/4277867e578c4f72b1d8c276fa5d6898, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/315d90da515a4ed1b3a435c5ae853310, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d6e65565d357402b9f979c1053c0885b] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=36.0 K 2024-11-11T02:31:32,750 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:32,750 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/b7f0eefc3b6d40fc9dad50bcbf6d3828, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/df1b3aeae8e74d9f9c94b707c61a4ed9, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/5100a46f7d6742c3b5f94ad7c834f059] 2024-11-11T02:31:32,750 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 4277867e578c4f72b1d8c276fa5d6898, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292287279 2024-11-11T02:31:32,750 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7f0eefc3b6d40fc9dad50bcbf6d3828, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292287279 2024-11-11T02:31:32,750 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 315d90da515a4ed1b3a435c5ae853310, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731292288399 2024-11-11T02:31:32,751 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting df1b3aeae8e74d9f9c94b707c61a4ed9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731292288399 2024-11-11T02:31:32,751 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5100a46f7d6742c3b5f94ad7c834f059, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731292290634 2024-11-11T02:31:32,751 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting d6e65565d357402b9f979c1053c0885b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731292290634 2024-11-11T02:31:32,757 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:32,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T02:31:32,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:32,758 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T02:31:32,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:32,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:32,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:32,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:32,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:32,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:32,758 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#B#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:32,759 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/8ee3429674514e42b6b9e11dfc784f32 is 50, key is test_row_0/B:col10/1731292290635/Put/seqid=0 2024-11-11T02:31:32,761 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411112a7e39f7a5014a0c91b66adfbebb5c40_e33fc31e962e33bc26f57b81ef4a0f99 store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:32,763 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411112a7e39f7a5014a0c91b66adfbebb5c40_e33fc31e962e33bc26f57b81ef4a0f99, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:32,763 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112a7e39f7a5014a0c91b66adfbebb5c40_e33fc31e962e33bc26f57b81ef4a0f99 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:32,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742003_1179 (size=12697) 2024-11-11T02:31:32,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742004_1180 (size=4469) 2024-11-11T02:31:32,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111080bffd3e7ce4d97956c00e47abeeace_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292291478/Put/seqid=0 2024-11-11T02:31:32,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742005_1181 (size=12454) 2024-11-11T02:31:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:32,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T02:31:32,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292292805, max timestamp=1731292292826Num uniques 859; 2024-11-11T02:31:32,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:31:32,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292292805, max timestamp=1731292292826Num uniques 859; 2024-11-11T02:31:32,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292292805, max timestamp=1731292292826Num uniques 859; 2024-11-11T02:31:32,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:31:32,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:31:32,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292352892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:32,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292352894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:32,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292352903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:32,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:32,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:32,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:33,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292353006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:33,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292353006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:33,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292353015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:33,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:33,176 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/8ee3429674514e42b6b9e11dfc784f32 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/8ee3429674514e42b6b9e11dfc784f32 2024-11-11T02:31:33,181 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#A#compaction#148 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:33,181 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/8768ca63596946b1a1c734fc66c66198 is 175, key is test_row_0/A:col10/1731292290635/Put/seqid=0 2024-11-11T02:31:33,182 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/B of e33fc31e962e33bc26f57b81ef4a0f99 into 8ee3429674514e42b6b9e11dfc784f32(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:33,182 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:33,182 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/B, priority=13, startTime=1731292292748; duration=0sec 2024-11-11T02:31:33,182 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:33,182 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:B 2024-11-11T02:31:33,182 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:33,184 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:33,184 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/C is initiating minor compaction (all files) 2024-11-11T02:31:33,184 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/C in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:33,184 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/1fa45d6fb3e440bb8ee52ca2ff0a2563, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/2c841996fafb4181a91aad43263d0cad, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/497c01272d954b5caa59501a0970bd7a] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=36.0 K 2024-11-11T02:31:33,185 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fa45d6fb3e440bb8ee52ca2ff0a2563, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292287279 2024-11-11T02:31:33,185 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c841996fafb4181a91aad43263d0cad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731292288399 2024-11-11T02:31:33,185 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 497c01272d954b5caa59501a0970bd7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731292290634 2024-11-11T02:31:33,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:33,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742006_1182 (size=31651) 2024-11-11T02:31:33,194 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111080bffd3e7ce4d97956c00e47abeeace_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111080bffd3e7ce4d97956c00e47abeeace_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:33,195 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#C#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:33,195 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/4fded1385294415992ef458a56b87add is 50, key is test_row_0/C:col10/1731292290635/Put/seqid=0 2024-11-11T02:31:33,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/42f14934a72846a0a490b5173e6cf2bd, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:33,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/42f14934a72846a0a490b5173e6cf2bd is 175, key is test_row_0/A:col10/1731292291478/Put/seqid=0 2024-11-11T02:31:33,205 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/8768ca63596946b1a1c734fc66c66198 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/8768ca63596946b1a1c734fc66c66198 2024-11-11T02:31:33,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742007_1183 (size=12697) 2024-11-11T02:31:33,212 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/A of e33fc31e962e33bc26f57b81ef4a0f99 into 8768ca63596946b1a1c734fc66c66198(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:33,212 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:33,212 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/A, priority=13, startTime=1731292292748; duration=0sec 2024-11-11T02:31:33,212 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:33,212 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:A 2024-11-11T02:31:33,214 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/4fded1385294415992ef458a56b87add as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/4fded1385294415992ef458a56b87add 2024-11-11T02:31:33,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:33,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292353216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:33,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:33,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292353216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:33,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742008_1184 (size=31255) 2024-11-11T02:31:33,222 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/C of e33fc31e962e33bc26f57b81ef4a0f99 into 4fded1385294415992ef458a56b87add(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:33,222 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:33,222 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/C, priority=13, startTime=1731292292748; duration=0sec 2024-11-11T02:31:33,222 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/42f14934a72846a0a490b5173e6cf2bd 2024-11-11T02:31:33,223 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:33,223 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:C 2024-11-11T02:31:33,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/133314cf2a814006917cd08700da733a is 50, key is test_row_0/B:col10/1731292291478/Put/seqid=0 2024-11-11T02:31:33,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742009_1185 (size=12301) 2024-11-11T02:31:33,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:33,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292353235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:33,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292353526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:33,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292353526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:33,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292353555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:33,639 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/133314cf2a814006917cd08700da733a 2024-11-11T02:31:33,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292353636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:33,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292353636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:33,641 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in e33fc31e962e33bc26f57b81ef4a0f99 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:33,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,642 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in e33fc31e962e33bc26f57b81ef4a0f99 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:33,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:33,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:33,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/86b4f099523748369a264202f317e71b is 50, key is test_row_0/C:col10/1731292291478/Put/seqid=0 2024-11-11T02:31:33,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742010_1186 (size=12301) 2024-11-11T02:31:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T02:31:34,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292354035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:34,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292354035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:34,052 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/86b4f099523748369a264202f317e71b 2024-11-11T02:31:34,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/42f14934a72846a0a490b5173e6cf2bd as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/42f14934a72846a0a490b5173e6cf2bd 2024-11-11T02:31:34,065 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/42f14934a72846a0a490b5173e6cf2bd, entries=150, sequenceid=264, filesize=30.5 K 2024-11-11T02:31:34,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/133314cf2a814006917cd08700da733a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/133314cf2a814006917cd08700da733a 2024-11-11T02:31:34,072 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/133314cf2a814006917cd08700da733a, entries=150, sequenceid=264, filesize=12.0 K 2024-11-11T02:31:34,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/86b4f099523748369a264202f317e71b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/86b4f099523748369a264202f317e71b 2024-11-11T02:31:34,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292354065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:34,081 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/86b4f099523748369a264202f317e71b, entries=150, sequenceid=264, filesize=12.0 K 2024-11-11T02:31:34,083 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for e33fc31e962e33bc26f57b81ef4a0f99 in 1326ms, sequenceid=264, compaction requested=false 2024-11-11T02:31:34,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:34,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:34,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-11T02:31:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-11T02:31:34,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-11T02:31:34,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3960 sec 2024-11-11T02:31:34,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.4000 sec 2024-11-11T02:31:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:35,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=161.02 KB heapSize=451.31 KB 2024-11-11T02:31:35,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731292292880, max timestamp=1731292295050Num uniques 1168; 2024-11-11T02:31:35,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=7688, suffix heap size=149440,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=168 2024-11-11T02:31:35,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:35,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:35,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:35,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731292292880, max timestamp=1731292295050Num uniques 1168; 2024-11-11T02:31:35,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:35,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:35,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:31:35,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=7688, suffix heap size=149440,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=168 2024-11-11T02:31:35,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111154c4a7fe27304f358f5ff215795225be_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292295050/Put/seqid=0 2024-11-11T02:31:35,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742011_1187 (size=12454) 2024-11-11T02:31:35,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292355156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292355158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:35,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292355163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:35,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:35,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292355266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:35,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292355276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:35,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292355286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:35,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292355485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:35,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292355486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:35,500 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:35,505 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111154c4a7fe27304f358f5ff215795225be_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111154c4a7fe27304f358f5ff215795225be_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:35,508 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/a8233c2202534da09ac85e964cdb65a5, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:35,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/a8233c2202534da09ac85e964cdb65a5 is 175, key is test_row_0/A:col10/1731292295050/Put/seqid=0 2024-11-11T02:31:35,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292355506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:35,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742012_1188 (size=31255) 2024-11-11T02:31:35,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292355657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:35,658 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in e33fc31e962e33bc26f57b81ef4a0f99 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:35,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292355666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:35,667 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in e33fc31e962e33bc26f57b81ef4a0f99 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:35,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292355796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:35,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292355805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:35,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:35,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292355816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T02:31:35,856 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:35,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-11T02:31:35,858 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:35,859 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:35,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T02:31:35,936 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/a8233c2202534da09ac85e964cdb65a5 2024-11-11T02:31:35,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/d7ed3abccfa6414abbe159065b1800a4 is 50, key is test_row_0/B:col10/1731292295050/Put/seqid=0 2024-11-11T02:31:35,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742013_1189 (size=12301) 2024-11-11T02:31:35,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/d7ed3abccfa6414abbe159065b1800a4 2024-11-11T02:31:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T02:31:35,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/8f96932906334341b97df3e87543e9cc is 50, key is test_row_0/C:col10/1731292295050/Put/seqid=0 2024-11-11T02:31:35,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742014_1190 (size=12301) 2024-11-11T02:31:35,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/8f96932906334341b97df3e87543e9cc 2024-11-11T02:31:35,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/a8233c2202534da09ac85e964cdb65a5 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/a8233c2202534da09ac85e964cdb65a5 2024-11-11T02:31:36,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/a8233c2202534da09ac85e964cdb65a5, entries=150, sequenceid=306, filesize=30.5 K 2024-11-11T02:31:36,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T02:31:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:36,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:36,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/d7ed3abccfa6414abbe159065b1800a4 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d7ed3abccfa6414abbe159065b1800a4 2024-11-11T02:31:36,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d7ed3abccfa6414abbe159065b1800a4, entries=150, sequenceid=306, filesize=12.0 K 2024-11-11T02:31:36,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/8f96932906334341b97df3e87543e9cc as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/8f96932906334341b97df3e87543e9cc 2024-11-11T02:31:36,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/8f96932906334341b97df3e87543e9cc, entries=150, sequenceid=306, filesize=12.0 K 2024-11-11T02:31:36,046 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-11T02:31:36,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~78.27 KB/80150, heapSize ~213.91 KB/219040, currentSize=100.63 KB/103050 for e33fc31e962e33bc26f57b81ef4a0f99 in 995ms, sequenceid=306, compaction requested=true 2024-11-11T02:31:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:36,048 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:36,048 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:36,050 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:36,050 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:36,050 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/A is initiating minor compaction (all files) 2024-11-11T02:31:36,050 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/B is initiating minor compaction (all files) 2024-11-11T02:31:36,050 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/A in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:36,050 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/B in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:36,050 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/8768ca63596946b1a1c734fc66c66198, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/42f14934a72846a0a490b5173e6cf2bd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/a8233c2202534da09ac85e964cdb65a5] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=92.0 K 2024-11-11T02:31:36,050 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/8ee3429674514e42b6b9e11dfc784f32, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/133314cf2a814006917cd08700da733a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d7ed3abccfa6414abbe159065b1800a4] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=36.4 K 2024-11-11T02:31:36,050 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:36,050 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/8768ca63596946b1a1c734fc66c66198, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/42f14934a72846a0a490b5173e6cf2bd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/a8233c2202534da09ac85e964cdb65a5] 2024-11-11T02:31:36,050 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ee3429674514e42b6b9e11dfc784f32, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731292290634 2024-11-11T02:31:36,051 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8768ca63596946b1a1c734fc66c66198, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731292290634 2024-11-11T02:31:36,051 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 133314cf2a814006917cd08700da733a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731292291476 2024-11-11T02:31:36,051 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42f14934a72846a0a490b5173e6cf2bd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731292291476 2024-11-11T02:31:36,051 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting d7ed3abccfa6414abbe159065b1800a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731292292901 2024-11-11T02:31:36,051 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8233c2202534da09ac85e964cdb65a5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731292292901 2024-11-11T02:31:36,067 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:36,068 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/cbf7c74d8d36444da7e11317c0e768b8 is 50, key is test_row_0/B:col10/1731292295050/Put/seqid=0 2024-11-11T02:31:36,074 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:36,086 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411119a71d94abbca46c3975c6b09b05323cf_e33fc31e962e33bc26f57b81ef4a0f99 store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:36,088 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411119a71d94abbca46c3975c6b09b05323cf_e33fc31e962e33bc26f57b81ef4a0f99, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:36,088 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119a71d94abbca46c3975c6b09b05323cf_e33fc31e962e33bc26f57b81ef4a0f99 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:36,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742015_1191 (size=12949) 2024-11-11T02:31:36,115 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/cbf7c74d8d36444da7e11317c0e768b8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/cbf7c74d8d36444da7e11317c0e768b8 2024-11-11T02:31:36,121 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/B of e33fc31e962e33bc26f57b81ef4a0f99 into cbf7c74d8d36444da7e11317c0e768b8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:36,121 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:36,121 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/B, priority=13, startTime=1731292296048; duration=0sec 2024-11-11T02:31:36,121 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:36,121 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:B 2024-11-11T02:31:36,121 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:36,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742016_1192 (size=4469) 2024-11-11T02:31:36,124 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#A#compaction#157 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:36,124 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/3a61d7bf4d5c4ae98837ad7a2151d580 is 175, key is test_row_0/A:col10/1731292295050/Put/seqid=0 2024-11-11T02:31:36,131 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:36,131 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/C is initiating minor compaction (all files) 2024-11-11T02:31:36,131 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/C in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:36,132 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/4fded1385294415992ef458a56b87add, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/86b4f099523748369a264202f317e71b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/8f96932906334341b97df3e87543e9cc] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=36.4 K 2024-11-11T02:31:36,132 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fded1385294415992ef458a56b87add, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731292290634 2024-11-11T02:31:36,133 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 86b4f099523748369a264202f317e71b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731292291476 2024-11-11T02:31:36,133 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f96932906334341b97df3e87543e9cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731292292901 2024-11-11T02:31:36,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742017_1193 (size=31903) 2024-11-11T02:31:36,157 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:36,158 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/d6c2b26e089a4aaaaeb247c499b8ede8 is 50, key is test_row_0/C:col10/1731292295050/Put/seqid=0 2024-11-11T02:31:36,161 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/3a61d7bf4d5c4ae98837ad7a2151d580 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/3a61d7bf4d5c4ae98837ad7a2151d580 2024-11-11T02:31:36,166 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/A of e33fc31e962e33bc26f57b81ef4a0f99 into 3a61d7bf4d5c4ae98837ad7a2151d580(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:36,166 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:36,166 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/A, priority=13, startTime=1731292296048; duration=0sec 2024-11-11T02:31:36,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T02:31:36,166 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:36,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:36,166 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:A 2024-11-11T02:31:36,166 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T02:31:36,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:36,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:36,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:36,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:36,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:36,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:36,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742018_1194 (size=12949) 2024-11-11T02:31:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T02:31:36,176 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/d6c2b26e089a4aaaaeb247c499b8ede8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/d6c2b26e089a4aaaaeb247c499b8ede8 2024-11-11T02:31:36,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111165c67271b1584460a5f8f138c0f86689_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292295156/Put/seqid=0 2024-11-11T02:31:36,186 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/C of e33fc31e962e33bc26f57b81ef4a0f99 into d6c2b26e089a4aaaaeb247c499b8ede8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:36,186 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:36,186 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/C, priority=13, startTime=1731292296048; duration=0sec 2024-11-11T02:31:36,187 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:36,187 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:C 2024-11-11T02:31:36,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742019_1195 (size=12454) 2024-11-11T02:31:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:36,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:36,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292356352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:36,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292356352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:36,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292356353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:36,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292356466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:36,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292356466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:36,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292356466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T02:31:36,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:36,611 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111165c67271b1584460a5f8f138c0f86689_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111165c67271b1584460a5f8f138c0f86689_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/f13cc79281e94e29b8c0986648a7f12b, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:36,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/f13cc79281e94e29b8c0986648a7f12b is 175, key is test_row_0/A:col10/1731292295156/Put/seqid=0 2024-11-11T02:31:36,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742020_1196 (size=31255) 2024-11-11T02:31:36,636 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/f13cc79281e94e29b8c0986648a7f12b 2024-11-11T02:31:36,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/b5c3a96667344fd8a25c4d6d423ef493 is 50, key is test_row_0/B:col10/1731292295156/Put/seqid=0 2024-11-11T02:31:36,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742021_1197 (size=12301) 2024-11-11T02:31:36,678 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/b5c3a96667344fd8a25c4d6d423ef493 2024-11-11T02:31:36,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292356676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:36,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292356676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:36,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292356676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:36,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/cbb5d2cb23ce4ae1936128b771cd8704 is 50, key is test_row_0/C:col10/1731292295156/Put/seqid=0 2024-11-11T02:31:36,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742022_1198 (size=12301) 2024-11-11T02:31:36,714 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/cbb5d2cb23ce4ae1936128b771cd8704 2024-11-11T02:31:36,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/f13cc79281e94e29b8c0986648a7f12b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f13cc79281e94e29b8c0986648a7f12b 2024-11-11T02:31:36,728 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f13cc79281e94e29b8c0986648a7f12b, entries=150, sequenceid=326, filesize=30.5 K 2024-11-11T02:31:36,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/b5c3a96667344fd8a25c4d6d423ef493 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b5c3a96667344fd8a25c4d6d423ef493 2024-11-11T02:31:36,737 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b5c3a96667344fd8a25c4d6d423ef493, entries=150, sequenceid=326, filesize=12.0 K 2024-11-11T02:31:36,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/cbb5d2cb23ce4ae1936128b771cd8704 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cbb5d2cb23ce4ae1936128b771cd8704 2024-11-11T02:31:36,747 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cbb5d2cb23ce4ae1936128b771cd8704, entries=150, sequenceid=326, filesize=12.0 K 2024-11-11T02:31:36,748 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e33fc31e962e33bc26f57b81ef4a0f99 in 582ms, sequenceid=326, compaction requested=false 2024-11-11T02:31:36,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:36,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:36,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-11T02:31:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-11T02:31:36,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-11T02:31:36,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 891 msec 2024-11-11T02:31:36,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 899 msec 2024-11-11T02:31:36,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:36,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T02:31:36,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:36,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:36,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:36,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:36,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:36,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T02:31:36,996 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:36,997 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:36,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-11T02:31:36,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T02:31:36,999 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:37,000 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:37,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:37,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ac40d7dfc64d486680b73e9c9c4a89cb_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292296350/Put/seqid=0 2024-11-11T02:31:37,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292357011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:37,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:37,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292357015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:37,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:37,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292357020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:37,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742023_1199 (size=14994) 2024-11-11T02:31:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T02:31:37,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292357126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:37,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:37,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292357126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:37,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:37,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292357136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:37,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T02:31:37,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:37,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T02:31:37,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:37,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T02:31:37,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292357338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:37,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292357338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:37,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292357345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:37,438 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:37,445 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ac40d7dfc64d486680b73e9c9c4a89cb_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ac40d7dfc64d486680b73e9c9c4a89cb_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:37,446 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/9119a84ee4ea4e4d83ef3c2d7eb77ce8, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:37,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/9119a84ee4ea4e4d83ef3c2d7eb77ce8 is 175, key is test_row_0/A:col10/1731292296350/Put/seqid=0 2024-11-11T02:31:37,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742024_1200 (size=39949) 2024-11-11T02:31:37,460 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/9119a84ee4ea4e4d83ef3c2d7eb77ce8 2024-11-11T02:31:37,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T02:31:37,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:37,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/f0103a3c5c9d41908367dfc2cffdcd25 is 50, key is test_row_0/B:col10/1731292296350/Put/seqid=0 2024-11-11T02:31:37,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742025_1201 (size=12301) 2024-11-11T02:31:37,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T02:31:37,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:37,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T02:31:37,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:37,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292357646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:37,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292357646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:37,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292357656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:37,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:37,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:37,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T02:31:37,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:37,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/f0103a3c5c9d41908367dfc2cffdcd25 2024-11-11T02:31:37,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/942cef60a5384ebca30f130837c7ddb0 is 50, key is test_row_0/C:col10/1731292296350/Put/seqid=0 2024-11-11T02:31:37,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T02:31:37,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:37,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:37,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742026_1202 (size=12301) 2024-11-11T02:31:37,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/942cef60a5384ebca30f130837c7ddb0 2024-11-11T02:31:37,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/9119a84ee4ea4e4d83ef3c2d7eb77ce8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9119a84ee4ea4e4d83ef3c2d7eb77ce8 2024-11-11T02:31:37,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9119a84ee4ea4e4d83ef3c2d7eb77ce8, entries=200, sequenceid=345, filesize=39.0 K 2024-11-11T02:31:37,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/f0103a3c5c9d41908367dfc2cffdcd25 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f0103a3c5c9d41908367dfc2cffdcd25 2024-11-11T02:31:37,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f0103a3c5c9d41908367dfc2cffdcd25, entries=150, sequenceid=345, filesize=12.0 K 2024-11-11T02:31:37,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/942cef60a5384ebca30f130837c7ddb0 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/942cef60a5384ebca30f130837c7ddb0 2024-11-11T02:31:37,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/942cef60a5384ebca30f130837c7ddb0, entries=150, sequenceid=345, filesize=12.0 K 2024-11-11T02:31:37,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e33fc31e962e33bc26f57b81ef4a0f99 in 971ms, sequenceid=345, compaction requested=true 2024-11-11T02:31:37,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:37,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:37,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:37,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:37,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T02:31:37,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:37,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T02:31:37,961 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:37,961 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:37,963 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:37,963 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/A is initiating minor compaction (all files) 2024-11-11T02:31:37,963 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/A in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,963 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/3a61d7bf4d5c4ae98837ad7a2151d580, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f13cc79281e94e29b8c0986648a7f12b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9119a84ee4ea4e4d83ef3c2d7eb77ce8] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=100.7 K 2024-11-11T02:31:37,963 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,963 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/3a61d7bf4d5c4ae98837ad7a2151d580, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f13cc79281e94e29b8c0986648a7f12b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9119a84ee4ea4e4d83ef3c2d7eb77ce8] 2024-11-11T02:31:37,963 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:37,963 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/C is initiating minor compaction (all files) 2024-11-11T02:31:37,964 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/C in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:37,964 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/d6c2b26e089a4aaaaeb247c499b8ede8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cbb5d2cb23ce4ae1936128b771cd8704, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/942cef60a5384ebca30f130837c7ddb0] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=36.7 K 2024-11-11T02:31:37,964 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting d6c2b26e089a4aaaaeb247c499b8ede8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731292292901 2024-11-11T02:31:37,964 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a61d7bf4d5c4ae98837ad7a2151d580, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731292292901 2024-11-11T02:31:37,964 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting cbb5d2cb23ce4ae1936128b771cd8704, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731292295145 2024-11-11T02:31:37,964 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting f13cc79281e94e29b8c0986648a7f12b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731292295145 2024-11-11T02:31:37,964 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 942cef60a5384ebca30f130837c7ddb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731292296335 2024-11-11T02:31:37,965 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9119a84ee4ea4e4d83ef3c2d7eb77ce8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731292296335 2024-11-11T02:31:37,974 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:37,979 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#C#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:37,979 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111237b234726ae40fbb3e9f2b420287668_e33fc31e962e33bc26f57b81ef4a0f99 store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:37,979 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/c76c0cd78cb54d83acabf5035b42e599 is 50, key is test_row_0/C:col10/1731292296350/Put/seqid=0 2024-11-11T02:31:37,981 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111237b234726ae40fbb3e9f2b420287668_e33fc31e962e33bc26f57b81ef4a0f99, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:37,981 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111237b234726ae40fbb3e9f2b420287668_e33fc31e962e33bc26f57b81ef4a0f99 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:37,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742027_1203 (size=13051) 2024-11-11T02:31:37,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742028_1204 (size=4469) 2024-11-11T02:31:38,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T02:31:38,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:38,077 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T02:31:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:38,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bec49c213f674560b1c0d94f08ab7078_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292297009/Put/seqid=0 2024-11-11T02:31:38,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742029_1205 (size=12454) 2024-11-11T02:31:38,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:38,114 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bec49c213f674560b1c0d94f08ab7078_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bec49c213f674560b1c0d94f08ab7078_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:38,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/f8649b98134a4ffabe2f08633df411a7, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:38,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/f8649b98134a4ffabe2f08633df411a7 is 175, key is test_row_0/A:col10/1731292297009/Put/seqid=0 2024-11-11T02:31:38,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742030_1206 (size=31255) 2024-11-11T02:31:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T02:31:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:38,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:38,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292358227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:38,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292358228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:38,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292358230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:38,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:38,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292358336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:38,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292358336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:38,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292358345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:38,396 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/c76c0cd78cb54d83acabf5035b42e599 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/c76c0cd78cb54d83acabf5035b42e599 2024-11-11T02:31:38,396 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#A#compaction#165 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:38,396 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/9ba5712eaa50450387904e390c7bbd92 is 175, key is test_row_0/A:col10/1731292296350/Put/seqid=0 2024-11-11T02:31:38,403 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/C of e33fc31e962e33bc26f57b81ef4a0f99 into c76c0cd78cb54d83acabf5035b42e599(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:38,403 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:38,403 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/C, priority=13, startTime=1731292297960; duration=0sec 2024-11-11T02:31:38,403 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:38,403 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:C 2024-11-11T02:31:38,403 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:38,404 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:38,404 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/B is initiating minor compaction (all files) 2024-11-11T02:31:38,404 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/B in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:38,405 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/cbf7c74d8d36444da7e11317c0e768b8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b5c3a96667344fd8a25c4d6d423ef493, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f0103a3c5c9d41908367dfc2cffdcd25] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=36.7 K 2024-11-11T02:31:38,405 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting cbf7c74d8d36444da7e11317c0e768b8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731292292901 2024-11-11T02:31:38,406 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting b5c3a96667344fd8a25c4d6d423ef493, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731292295145 2024-11-11T02:31:38,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742031_1207 (size=32005) 2024-11-11T02:31:38,407 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting f0103a3c5c9d41908367dfc2cffdcd25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731292296335 2024-11-11T02:31:38,411 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/9ba5712eaa50450387904e390c7bbd92 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9ba5712eaa50450387904e390c7bbd92 2024-11-11T02:31:38,417 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#B#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:38,417 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/e75bf71fc83e485985686708e0a8f9cc is 50, key is test_row_0/B:col10/1731292296350/Put/seqid=0 2024-11-11T02:31:38,420 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/A of e33fc31e962e33bc26f57b81ef4a0f99 into 9ba5712eaa50450387904e390c7bbd92(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:38,420 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:38,420 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/A, priority=13, startTime=1731292297960; duration=0sec 2024-11-11T02:31:38,421 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:38,421 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:A 2024-11-11T02:31:38,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742032_1208 (size=13051) 2024-11-11T02:31:38,525 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/f8649b98134a4ffabe2f08633df411a7 2024-11-11T02:31:38,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/76357a7eebb9470092d1866f9bf32827 is 50, key is test_row_0/B:col10/1731292297009/Put/seqid=0 2024-11-11T02:31:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292358546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292358546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:38,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742033_1209 (size=12301) 2024-11-11T02:31:38,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292358565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:38,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:38,843 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/e75bf71fc83e485985686708e0a8f9cc as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/e75bf71fc83e485985686708e0a8f9cc 2024-11-11T02:31:38,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292358855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:38,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292358856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:38,863 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/B of e33fc31e962e33bc26f57b81ef4a0f99 into e75bf71fc83e485985686708e0a8f9cc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:38,863 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:38,863 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/B, priority=13, startTime=1731292297960; duration=0sec 2024-11-11T02:31:38,863 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:38,863 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:B 2024-11-11T02:31:38,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292358876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:38,964 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/76357a7eebb9470092d1866f9bf32827 2024-11-11T02:31:38,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/1f6e71fcc9b54404a2b268573bf945a9 is 50, key is test_row_0/C:col10/1731292297009/Put/seqid=0 2024-11-11T02:31:39,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742034_1210 (size=12301) 2024-11-11T02:31:39,009 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/1f6e71fcc9b54404a2b268573bf945a9 2024-11-11T02:31:39,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/f8649b98134a4ffabe2f08633df411a7 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f8649b98134a4ffabe2f08633df411a7 2024-11-11T02:31:39,026 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f8649b98134a4ffabe2f08633df411a7, entries=150, sequenceid=359, filesize=30.5 K 2024-11-11T02:31:39,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/76357a7eebb9470092d1866f9bf32827 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/76357a7eebb9470092d1866f9bf32827 2024-11-11T02:31:39,043 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/76357a7eebb9470092d1866f9bf32827, entries=150, sequenceid=359, filesize=12.0 K 2024-11-11T02:31:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/1f6e71fcc9b54404a2b268573bf945a9 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/1f6e71fcc9b54404a2b268573bf945a9 2024-11-11T02:31:39,049 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/1f6e71fcc9b54404a2b268573bf945a9, entries=150, sequenceid=359, filesize=12.0 K 2024-11-11T02:31:39,054 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e33fc31e962e33bc26f57b81ef4a0f99 in 977ms, sequenceid=359, compaction requested=false 2024-11-11T02:31:39,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:39,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:39,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-11T02:31:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-11T02:31:39,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-11T02:31:39,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0570 sec 2024-11-11T02:31:39,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.0640 sec 2024-11-11T02:31:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T02:31:39,146 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:39,146 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-11T02:31:39,149 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T02:31:39,149 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:39,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T02:31:39,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T02:31:39,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:39,303 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T02:31:39,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:39,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:39,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:39,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:39,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:39,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:39,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bbd5b0220393411ea61d69a9a4ba61c6_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292298202/Put/seqid=0 2024-11-11T02:31:39,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742035_1211 (size=12454) 2024-11-11T02:31:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:39,345 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bbd5b0220393411ea61d69a9a4ba61c6_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bbd5b0220393411ea61d69a9a4ba61c6_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:39,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/992c3bd8b9064b69a8f141bb5f9cec76, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:39,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/992c3bd8b9064b69a8f141bb5f9cec76 is 175, key is test_row_0/A:col10/1731292298202/Put/seqid=0 2024-11-11T02:31:39,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742036_1212 (size=31255) 2024-11-11T02:31:39,363 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/992c3bd8b9064b69a8f141bb5f9cec76 2024-11-11T02:31:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:39,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:39,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/f061d5f00a0443c9abcfcd44f79acf22 is 50, key is test_row_0/B:col10/1731292298202/Put/seqid=0 2024-11-11T02:31:39,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742037_1213 (size=12301) 2024-11-11T02:31:39,393 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/f061d5f00a0443c9abcfcd44f79acf22 2024-11-11T02:31:39,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292359392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:39,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292359393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:39,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292359397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:39,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/bdd33af4341b4cfcbb22ee8b4420bb6b is 50, key is test_row_0/C:col10/1731292298202/Put/seqid=0 2024-11-11T02:31:39,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742038_1214 (size=12301) 2024-11-11T02:31:39,425 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/bdd33af4341b4cfcbb22ee8b4420bb6b 2024-11-11T02:31:39,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/992c3bd8b9064b69a8f141bb5f9cec76 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/992c3bd8b9064b69a8f141bb5f9cec76 2024-11-11T02:31:39,439 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/992c3bd8b9064b69a8f141bb5f9cec76, entries=150, sequenceid=380, filesize=30.5 K 2024-11-11T02:31:39,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/f061d5f00a0443c9abcfcd44f79acf22 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f061d5f00a0443c9abcfcd44f79acf22 2024-11-11T02:31:39,447 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f061d5f00a0443c9abcfcd44f79acf22, entries=150, sequenceid=380, filesize=12.0 K 2024-11-11T02:31:39,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/bdd33af4341b4cfcbb22ee8b4420bb6b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/bdd33af4341b4cfcbb22ee8b4420bb6b 2024-11-11T02:31:39,463 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/bdd33af4341b4cfcbb22ee8b4420bb6b, entries=150, sequenceid=380, filesize=12.0 K 2024-11-11T02:31:39,465 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e33fc31e962e33bc26f57b81ef4a0f99 in 162ms, sequenceid=380, compaction requested=true 2024-11-11T02:31:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-11T02:31:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T02:31:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-11T02:31:39,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-11T02:31:39,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 317 msec 2024-11-11T02:31:39,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 325 msec 2024-11-11T02:31:39,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T02:31:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:39,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:39,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:39,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:39,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:39,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d5e6665a946848b68389fe744b7d85eb_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292299506/Put/seqid=0 2024-11-11T02:31:39,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742039_1215 (size=17534) 2024-11-11T02:31:39,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292359557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:39,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292359562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:39,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292359564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:39,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:39,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292359676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:39,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731292359676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:39,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292359676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292359676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:39,679 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in e33fc31e962e33bc26f57b81ef4a0f99 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:39,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1731292359718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:39,722 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in e33fc31e962e33bc26f57b81ef4a0f99 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:39,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T02:31:39,777 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:39,777 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-11T02:31:39,780 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:39,780 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:39,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T02:31:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T02:31:39,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292359886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292359886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:39,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292359886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:39,923 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:39,927 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d5e6665a946848b68389fe744b7d85eb_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5e6665a946848b68389fe744b7d85eb_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:39,928 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/31d50d85ddca43438fede828255e7ab8, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:39,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/31d50d85ddca43438fede828255e7ab8 is 175, key is test_row_0/A:col10/1731292299506/Put/seqid=0 2024-11-11T02:31:39,933 INFO [Thread-597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-597. 2024-11-11T02:31:39,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742040_1216 (size=48639) 2024-11-11T02:31:39,933 DEBUG [Thread-597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:39,933 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:39,933 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:39,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:39,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-11T02:31:39,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:39,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:39,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:39,935 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/31d50d85ddca43438fede828255e7ab8 2024-11-11T02:31:39,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:39,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:39,935 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-11-11T02:31:39,935 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:39,935 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:39,935 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:39,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:39,938 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-11-11T02:31:39,938 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:39,938 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:39,939 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:39,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:39,943 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-11-11T02:31:39,943 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:39,943 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:39,943 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:39,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:39,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/68da243be80f47f099ba165219e2bc47 is 50, key is test_row_0/B:col10/1731292299506/Put/seqid=0 2024-11-11T02:31:39,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742041_1217 (size=12301) 2024-11-11T02:31:40,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-11T02:31:40,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:40,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:40,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:40,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T02:31:40,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:40,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:40,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292360196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292360196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292360196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:40,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:40,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:40,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:40,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:40,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:40,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:40,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:40,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:40,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:40,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-11T02:31:40,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:40,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:40,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:40,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/68da243be80f47f099ba165219e2bc47 2024-11-11T02:31:40,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/71b882058d5044f0b5af35edb1e29ed8 is 50, key is test_row_0/C:col10/1731292299506/Put/seqid=0 2024-11-11T02:31:40,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742042_1218 (size=12301) 2024-11-11T02:31:40,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-11T02:31:40,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:40,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:40,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:40,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T02:31:40,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-11T02:31:40,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:40,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:40,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:40,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-11T02:31:40,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:40,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:40,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:40,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:40,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:40,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:40,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49434 deadline: 1731292360705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731292360705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49418 deadline: 1731292360705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:31:40,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/71b882058d5044f0b5af35edb1e29ed8 2024-11-11T02:31:40,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/31d50d85ddca43438fede828255e7ab8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/31d50d85ddca43438fede828255e7ab8 2024-11-11T02:31:40,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/31d50d85ddca43438fede828255e7ab8, entries=250, sequenceid=396, filesize=47.5 K 2024-11-11T02:31:40,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/68da243be80f47f099ba165219e2bc47 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/68da243be80f47f099ba165219e2bc47 2024-11-11T02:31:40,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/68da243be80f47f099ba165219e2bc47, entries=150, sequenceid=396, filesize=12.0 K 2024-11-11T02:31:40,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/71b882058d5044f0b5af35edb1e29ed8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/71b882058d5044f0b5af35edb1e29ed8 2024-11-11T02:31:40,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/71b882058d5044f0b5af35edb1e29ed8, entries=150, sequenceid=396, filesize=12.0 K 2024-11-11T02:31:40,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e33fc31e962e33bc26f57b81ef4a0f99 in 1283ms, sequenceid=396, compaction requested=true 2024-11-11T02:31:40,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:40,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:40,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:40,792 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:40,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:40,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:40,792 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:40,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e33fc31e962e33bc26f57b81ef4a0f99:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:40,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:40,793 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:40,793 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:40,793 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/B is initiating minor compaction (all files) 2024-11-11T02:31:40,794 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/A is initiating minor compaction (all files) 2024-11-11T02:31:40,794 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/B in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:40,794 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/A in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:40,794 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/e75bf71fc83e485985686708e0a8f9cc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/76357a7eebb9470092d1866f9bf32827, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f061d5f00a0443c9abcfcd44f79acf22, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/68da243be80f47f099ba165219e2bc47] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=48.8 K 2024-11-11T02:31:40,794 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9ba5712eaa50450387904e390c7bbd92, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f8649b98134a4ffabe2f08633df411a7, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/992c3bd8b9064b69a8f141bb5f9cec76, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/31d50d85ddca43438fede828255e7ab8] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=139.8 K 2024-11-11T02:31:40,794 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:40,794 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9ba5712eaa50450387904e390c7bbd92, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f8649b98134a4ffabe2f08633df411a7, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/992c3bd8b9064b69a8f141bb5f9cec76, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/31d50d85ddca43438fede828255e7ab8] 2024-11-11T02:31:40,794 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting e75bf71fc83e485985686708e0a8f9cc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731292296335 2024-11-11T02:31:40,794 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ba5712eaa50450387904e390c7bbd92, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731292296335 2024-11-11T02:31:40,794 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 76357a7eebb9470092d1866f9bf32827, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731292297009 2024-11-11T02:31:40,794 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8649b98134a4ffabe2f08633df411a7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731292297009 2024-11-11T02:31:40,794 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting f061d5f00a0443c9abcfcd44f79acf22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731292298202 2024-11-11T02:31:40,795 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 992c3bd8b9064b69a8f141bb5f9cec76, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731292298202 2024-11-11T02:31:40,795 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 68da243be80f47f099ba165219e2bc47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731292299394 2024-11-11T02:31:40,795 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31d50d85ddca43438fede828255e7ab8, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731292299390 2024-11-11T02:31:40,807 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#B#compaction#177 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:40,807 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/0a76dc5f52ce48e3a7ebc487236b990c is 50, key is test_row_0/B:col10/1731292299506/Put/seqid=0 2024-11-11T02:31:40,807 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:40,810 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110706c79bae7c4363a27b8bd9cd33d166_e33fc31e962e33bc26f57b81ef4a0f99 store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:40,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742043_1219 (size=13187) 2024-11-11T02:31:40,842 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110706c79bae7c4363a27b8bd9cd33d166_e33fc31e962e33bc26f57b81ef4a0f99, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:40,843 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110706c79bae7c4363a27b8bd9cd33d166_e33fc31e962e33bc26f57b81ef4a0f99 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:40,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742044_1220 (size=4469) 2024-11-11T02:31:40,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-11T02:31:40,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:40,858 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T02:31:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118059c6f2f7144fe59b99678e717ff29f_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292299562/Put/seqid=0 2024-11-11T02:31:40,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742045_1221 (size=12454) 2024-11-11T02:31:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T02:31:41,217 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/0a76dc5f52ce48e3a7ebc487236b990c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/0a76dc5f52ce48e3a7ebc487236b990c 2024-11-11T02:31:41,223 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/B of e33fc31e962e33bc26f57b81ef4a0f99 into 0a76dc5f52ce48e3a7ebc487236b990c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:41,223 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:41,223 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/B, priority=12, startTime=1731292300792; duration=0sec 2024-11-11T02:31:41,223 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:41,223 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:B 2024-11-11T02:31:41,223 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:31:41,224 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:31:41,224 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): e33fc31e962e33bc26f57b81ef4a0f99/C is initiating minor compaction (all files) 2024-11-11T02:31:41,224 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e33fc31e962e33bc26f57b81ef4a0f99/C in TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:41,225 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/c76c0cd78cb54d83acabf5035b42e599, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/1f6e71fcc9b54404a2b268573bf945a9, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/bdd33af4341b4cfcbb22ee8b4420bb6b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/71b882058d5044f0b5af35edb1e29ed8] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp, totalSize=48.8 K 2024-11-11T02:31:41,225 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting c76c0cd78cb54d83acabf5035b42e599, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731292296335 2024-11-11T02:31:41,225 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f6e71fcc9b54404a2b268573bf945a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731292297009 2024-11-11T02:31:41,226 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting bdd33af4341b4cfcbb22ee8b4420bb6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731292298202 2024-11-11T02:31:41,226 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 71b882058d5044f0b5af35edb1e29ed8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731292299394 2024-11-11T02:31:41,235 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:41,236 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/e5719440ee81498d809010838cb7274c is 50, key is test_row_0/C:col10/1731292299506/Put/seqid=0 2024-11-11T02:31:41,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742046_1222 (size=13187) 2024-11-11T02:31:41,254 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e33fc31e962e33bc26f57b81ef4a0f99#A#compaction#178 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:41,255 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/f783dcf146ab43eba4e742a1f230a32c is 175, key is test_row_0/A:col10/1731292299506/Put/seqid=0 2024-11-11T02:31:41,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742047_1223 (size=32141) 2024-11-11T02:31:41,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:41,282 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118059c6f2f7144fe59b99678e717ff29f_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118059c6f2f7144fe59b99678e717ff29f_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:41,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/1c46539c38ca47f198edd8e6317b276d, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:41,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/1c46539c38ca47f198edd8e6317b276d is 175, key is test_row_0/A:col10/1731292299562/Put/seqid=0 2024-11-11T02:31:41,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742048_1224 (size=31255) 2024-11-11T02:31:41,652 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/e5719440ee81498d809010838cb7274c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/e5719440ee81498d809010838cb7274c 2024-11-11T02:31:41,657 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/C of e33fc31e962e33bc26f57b81ef4a0f99 into e5719440ee81498d809010838cb7274c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:41,657 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:41,657 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/C, priority=12, startTime=1731292300792; duration=0sec 2024-11-11T02:31:41,657 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:41,657 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:C 2024-11-11T02:31:41,664 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/f783dcf146ab43eba4e742a1f230a32c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f783dcf146ab43eba4e742a1f230a32c 2024-11-11T02:31:41,669 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e33fc31e962e33bc26f57b81ef4a0f99/A of e33fc31e962e33bc26f57b81ef4a0f99 into f783dcf146ab43eba4e742a1f230a32c(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:41,669 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:41,669 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., storeName=e33fc31e962e33bc26f57b81ef4a0f99/A, priority=12, startTime=1731292300792; duration=0sec 2024-11-11T02:31:41,669 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:41,669 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e33fc31e962e33bc26f57b81ef4a0f99:A 2024-11-11T02:31:41,688 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=412, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/1c46539c38ca47f198edd8e6317b276d 2024-11-11T02:31:41,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/da34b0f1d61e452c9d56340c56606ada is 50, key is test_row_0/B:col10/1731292299562/Put/seqid=0 2024-11-11T02:31:41,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742049_1225 (size=12301) 2024-11-11T02:31:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:41,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. as already flushing 2024-11-11T02:31:41,717 INFO [Thread-586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-586. 2024-11-11T02:31:41,717 INFO [Thread-590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-590. 2024-11-11T02:31:41,717 DEBUG [Thread-590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:41,717 DEBUG [Thread-586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:41,717 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:41,717 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:41,717 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:41,717 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:41,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:41,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:41,727 INFO [Thread-588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-588. 2024-11-11T02:31:41,727 DEBUG [Thread-588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:41,727 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:41,727 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:41,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T02:31:42,102 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/da34b0f1d61e452c9d56340c56606ada 2024-11-11T02:31:42,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/e531d186af624669857b18219ef46fe9 is 50, key is test_row_0/C:col10/1731292299562/Put/seqid=0 2024-11-11T02:31:42,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742050_1226 (size=12301) 2024-11-11T02:31:42,519 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/e531d186af624669857b18219ef46fe9 2024-11-11T02:31:42,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/1c46539c38ca47f198edd8e6317b276d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/1c46539c38ca47f198edd8e6317b276d 2024-11-11T02:31:42,532 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/1c46539c38ca47f198edd8e6317b276d, entries=150, sequenceid=412, filesize=30.5 K 2024-11-11T02:31:42,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/da34b0f1d61e452c9d56340c56606ada as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/da34b0f1d61e452c9d56340c56606ada 2024-11-11T02:31:42,538 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/da34b0f1d61e452c9d56340c56606ada, entries=150, sequenceid=412, filesize=12.0 K 2024-11-11T02:31:42,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/e531d186af624669857b18219ef46fe9 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/e531d186af624669857b18219ef46fe9 2024-11-11T02:31:42,542 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/e531d186af624669857b18219ef46fe9, entries=150, sequenceid=412, filesize=12.0 K 2024-11-11T02:31:42,543 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=20.13 KB/20610 for e33fc31e962e33bc26f57b81ef4a0f99 in 1685ms, sequenceid=412, compaction requested=false 2024-11-11T02:31:42,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for e33fc31e962e33bc26f57b81ef4a0f99: 2024-11-11T02:31:42,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:42,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-11T02:31:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-11T02:31:42,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-11T02:31:42,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7640 sec 2024-11-11T02:31:42,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.7700 sec 2024-11-11T02:31:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T02:31:43,937 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:46,013 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T02:31:49,700 INFO [Thread-592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-592. 2024-11-11T02:31:49,700 DEBUG [Thread-592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:49,700 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:49,700 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:49,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:49,750 INFO [Thread-594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-594. 2024-11-11T02:31:49,750 DEBUG [Thread-594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:31:49,750 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:49,751 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:49,751 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:49,751 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T02:31:49,751 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-11T02:31:49,751 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-11T02:31:49,752 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-11T02:31:49,752 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-11T02:31:49,752 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-11T02:31:49,752 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T02:31:49,752 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2899 2024-11-11T02:31:49,752 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2877 2024-11-11T02:31:49,752 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T02:31:49,752 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1244 2024-11-11T02:31:49,752 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3728 rows 2024-11-11T02:31:49,752 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1246 2024-11-11T02:31:49,752 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3734 rows 2024-11-11T02:31:49,752 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:31:49,753 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:31:49,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:49,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:49,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:31:49,755 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T02:31:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T02:31:49,760 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292309760"}]},"ts":"1731292309760"} 2024-11-11T02:31:49,762 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T02:31:49,762 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T02:31:49,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T02:31:49,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, UNASSIGN}] 2024-11-11T02:31:49,766 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, UNASSIGN 2024-11-11T02:31:49,767 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=e33fc31e962e33bc26f57b81ef4a0f99, regionState=CLOSING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:31:49,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, UNASSIGN because future has completed 2024-11-11T02:31:49,770 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T02:31:49,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:31:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T02:31:49,923 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:49,923 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T02:31:49,924 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing e33fc31e962e33bc26f57b81ef4a0f99, disabling compactions & flushes 2024-11-11T02:31:49,924 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:49,924 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:49,924 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. after waiting 0 ms 2024-11-11T02:31:49,924 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:49,924 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing e33fc31e962e33bc26f57b81ef4a0f99 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T02:31:49,925 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=A 2024-11-11T02:31:49,925 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:49,925 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=B 2024-11-11T02:31:49,925 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:49,925 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e33fc31e962e33bc26f57b81ef4a0f99, store=C 2024-11-11T02:31:49,925 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:49,935 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d36f2875794d49a985b8811c711e6799_e33fc31e962e33bc26f57b81ef4a0f99 is 50, key is test_row_0/A:col10/1731292309746/Put/seqid=0 2024-11-11T02:31:49,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742051_1227 (size=9914) 2024-11-11T02:31:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T02:31:50,340 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:50,349 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d36f2875794d49a985b8811c711e6799_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d36f2875794d49a985b8811c711e6799_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:50,350 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/ad6afb8d1b584ed885de7400005822c6, store: [table=TestAcidGuarantees family=A region=e33fc31e962e33bc26f57b81ef4a0f99] 2024-11-11T02:31:50,351 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/ad6afb8d1b584ed885de7400005822c6 is 175, key is test_row_0/A:col10/1731292309746/Put/seqid=0 2024-11-11T02:31:50,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742052_1228 (size=22561) 2024-11-11T02:31:50,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T02:31:50,759 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/ad6afb8d1b584ed885de7400005822c6 2024-11-11T02:31:50,773 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/3f4e8821cbb34da4bb7306ac21b09960 is 50, key is test_row_0/B:col10/1731292309746/Put/seqid=0 2024-11-11T02:31:50,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742053_1229 (size=9857) 2024-11-11T02:31:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T02:31:51,179 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/3f4e8821cbb34da4bb7306ac21b09960 2024-11-11T02:31:51,194 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/9157898565184eaca9e6488465c95b5c is 50, key is test_row_0/C:col10/1731292309746/Put/seqid=0 2024-11-11T02:31:51,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742054_1230 (size=9857) 2024-11-11T02:31:51,599 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/9157898565184eaca9e6488465c95b5c 2024-11-11T02:31:51,609 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/A/ad6afb8d1b584ed885de7400005822c6 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ad6afb8d1b584ed885de7400005822c6 2024-11-11T02:31:51,614 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ad6afb8d1b584ed885de7400005822c6, entries=100, sequenceid=423, filesize=22.0 K 2024-11-11T02:31:51,615 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/B/3f4e8821cbb34da4bb7306ac21b09960 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/3f4e8821cbb34da4bb7306ac21b09960 2024-11-11T02:31:51,619 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/3f4e8821cbb34da4bb7306ac21b09960, entries=100, sequenceid=423, filesize=9.6 K 2024-11-11T02:31:51,620 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/.tmp/C/9157898565184eaca9e6488465c95b5c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/9157898565184eaca9e6488465c95b5c 2024-11-11T02:31:51,624 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/9157898565184eaca9e6488465c95b5c, entries=100, sequenceid=423, filesize=9.6 K 2024-11-11T02:31:51,625 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e33fc31e962e33bc26f57b81ef4a0f99 in 1701ms, sequenceid=423, compaction requested=true 2024-11-11T02:31:51,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/d85fe9679127423c82bc55420716f7d4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/bd0605a37ce144afb72d378d5f2a7772, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/141ce89107304c329ca3fa47760320fa, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/cf92d79bd2c04dabbd08a82c8913c80b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ab973ef1dea8426da11ceb15cdf9183b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/71db1fcc23cf46bdab491d4c1fd53665, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9ae47f5978534435bfa813eb2b496479, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/344626f82d8347b3952ecc80fac626fc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/229672d84e2346eb8466ace05d3cc792, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/e73e539eadba47dda202182b403626b7, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/4f5c2ae4d2ee40bb9bf9e42f14cc24ca, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ca1ee4764efd48daa54a1dfd150da053, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/5696ab62539e484da59ee91a0bbcc575, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/b7f0eefc3b6d40fc9dad50bcbf6d3828, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/df1b3aeae8e74d9f9c94b707c61a4ed9, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/8768ca63596946b1a1c734fc66c66198, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/5100a46f7d6742c3b5f94ad7c834f059, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/42f14934a72846a0a490b5173e6cf2bd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/3a61d7bf4d5c4ae98837ad7a2151d580, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/a8233c2202534da09ac85e964cdb65a5, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f13cc79281e94e29b8c0986648a7f12b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9119a84ee4ea4e4d83ef3c2d7eb77ce8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9ba5712eaa50450387904e390c7bbd92, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f8649b98134a4ffabe2f08633df411a7, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/992c3bd8b9064b69a8f141bb5f9cec76, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/31d50d85ddca43438fede828255e7ab8] to archive 2024-11-11T02:31:51,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:31:51,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/d85fe9679127423c82bc55420716f7d4 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/d85fe9679127423c82bc55420716f7d4 2024-11-11T02:31:51,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/bd0605a37ce144afb72d378d5f2a7772 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/bd0605a37ce144afb72d378d5f2a7772 2024-11-11T02:31:51,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/141ce89107304c329ca3fa47760320fa to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/141ce89107304c329ca3fa47760320fa 2024-11-11T02:31:51,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/cf92d79bd2c04dabbd08a82c8913c80b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/cf92d79bd2c04dabbd08a82c8913c80b 2024-11-11T02:31:51,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ab973ef1dea8426da11ceb15cdf9183b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ab973ef1dea8426da11ceb15cdf9183b 2024-11-11T02:31:51,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/71db1fcc23cf46bdab491d4c1fd53665 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/71db1fcc23cf46bdab491d4c1fd53665 2024-11-11T02:31:51,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9ae47f5978534435bfa813eb2b496479 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9ae47f5978534435bfa813eb2b496479 2024-11-11T02:31:51,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/344626f82d8347b3952ecc80fac626fc to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/344626f82d8347b3952ecc80fac626fc 2024-11-11T02:31:51,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/229672d84e2346eb8466ace05d3cc792 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/229672d84e2346eb8466ace05d3cc792 2024-11-11T02:31:51,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/e73e539eadba47dda202182b403626b7 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/e73e539eadba47dda202182b403626b7 2024-11-11T02:31:51,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/4f5c2ae4d2ee40bb9bf9e42f14cc24ca to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/4f5c2ae4d2ee40bb9bf9e42f14cc24ca 2024-11-11T02:31:51,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ca1ee4764efd48daa54a1dfd150da053 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ca1ee4764efd48daa54a1dfd150da053 2024-11-11T02:31:51,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/5696ab62539e484da59ee91a0bbcc575 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/5696ab62539e484da59ee91a0bbcc575 2024-11-11T02:31:51,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/b7f0eefc3b6d40fc9dad50bcbf6d3828 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/b7f0eefc3b6d40fc9dad50bcbf6d3828 2024-11-11T02:31:51,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/df1b3aeae8e74d9f9c94b707c61a4ed9 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/df1b3aeae8e74d9f9c94b707c61a4ed9 2024-11-11T02:31:51,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/8768ca63596946b1a1c734fc66c66198 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/8768ca63596946b1a1c734fc66c66198 2024-11-11T02:31:51,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/5100a46f7d6742c3b5f94ad7c834f059 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/5100a46f7d6742c3b5f94ad7c834f059 2024-11-11T02:31:51,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/42f14934a72846a0a490b5173e6cf2bd to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/42f14934a72846a0a490b5173e6cf2bd 2024-11-11T02:31:51,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/3a61d7bf4d5c4ae98837ad7a2151d580 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/3a61d7bf4d5c4ae98837ad7a2151d580 2024-11-11T02:31:51,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/a8233c2202534da09ac85e964cdb65a5 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/a8233c2202534da09ac85e964cdb65a5 2024-11-11T02:31:51,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f13cc79281e94e29b8c0986648a7f12b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f13cc79281e94e29b8c0986648a7f12b 2024-11-11T02:31:51,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9119a84ee4ea4e4d83ef3c2d7eb77ce8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9119a84ee4ea4e4d83ef3c2d7eb77ce8 2024-11-11T02:31:51,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9ba5712eaa50450387904e390c7bbd92 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/9ba5712eaa50450387904e390c7bbd92 2024-11-11T02:31:51,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f8649b98134a4ffabe2f08633df411a7 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f8649b98134a4ffabe2f08633df411a7 2024-11-11T02:31:51,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/992c3bd8b9064b69a8f141bb5f9cec76 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/992c3bd8b9064b69a8f141bb5f9cec76 2024-11-11T02:31:51,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/31d50d85ddca43438fede828255e7ab8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/31d50d85ddca43438fede828255e7ab8 2024-11-11T02:31:51,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/28fbcd689a3f43b7a110673ea3093d82, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/49d73717bf0d4717969ab51f2fb8967c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/0477c7c3d001437686dca46c9f7d8d61, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/4c1bf6b2c5514a56acf928573c59ad9c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/572e04d88de24381945dfe596270a536, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b2f87cc3fa824a5c9b9534af6594c5cc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f2c0ad023be34275ad8530de9fb2c554, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f93af1f14ec84248ab453de3dbb310d0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/1959499c07734bc7b613ae569ccf903f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d00b3721396b4ef6b247a434925587aa, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/c8b4de2ccfe542e7a3fbdcda36a02d4e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/8fb9473a6f774b65aa9cb8cd198b4202, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/4277867e578c4f72b1d8c276fa5d6898, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b78a867e876d4fd3914e3af416ea161f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/315d90da515a4ed1b3a435c5ae853310, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/8ee3429674514e42b6b9e11dfc784f32, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d6e65565d357402b9f979c1053c0885b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/133314cf2a814006917cd08700da733a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/cbf7c74d8d36444da7e11317c0e768b8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d7ed3abccfa6414abbe159065b1800a4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b5c3a96667344fd8a25c4d6d423ef493, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/e75bf71fc83e485985686708e0a8f9cc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f0103a3c5c9d41908367dfc2cffdcd25, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/76357a7eebb9470092d1866f9bf32827, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f061d5f00a0443c9abcfcd44f79acf22, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/68da243be80f47f099ba165219e2bc47] to archive 2024-11-11T02:31:51,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:31:51,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/28fbcd689a3f43b7a110673ea3093d82 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/28fbcd689a3f43b7a110673ea3093d82 2024-11-11T02:31:51,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/49d73717bf0d4717969ab51f2fb8967c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/49d73717bf0d4717969ab51f2fb8967c 2024-11-11T02:31:51,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/0477c7c3d001437686dca46c9f7d8d61 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/0477c7c3d001437686dca46c9f7d8d61 2024-11-11T02:31:51,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/4c1bf6b2c5514a56acf928573c59ad9c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/4c1bf6b2c5514a56acf928573c59ad9c 2024-11-11T02:31:51,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/572e04d88de24381945dfe596270a536 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/572e04d88de24381945dfe596270a536 2024-11-11T02:31:51,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b2f87cc3fa824a5c9b9534af6594c5cc to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b2f87cc3fa824a5c9b9534af6594c5cc 2024-11-11T02:31:51,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f2c0ad023be34275ad8530de9fb2c554 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f2c0ad023be34275ad8530de9fb2c554 2024-11-11T02:31:51,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f93af1f14ec84248ab453de3dbb310d0 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f93af1f14ec84248ab453de3dbb310d0 2024-11-11T02:31:51,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/1959499c07734bc7b613ae569ccf903f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/1959499c07734bc7b613ae569ccf903f 2024-11-11T02:31:51,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d00b3721396b4ef6b247a434925587aa to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d00b3721396b4ef6b247a434925587aa 2024-11-11T02:31:51,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/c8b4de2ccfe542e7a3fbdcda36a02d4e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/c8b4de2ccfe542e7a3fbdcda36a02d4e 2024-11-11T02:31:51,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/8fb9473a6f774b65aa9cb8cd198b4202 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/8fb9473a6f774b65aa9cb8cd198b4202 2024-11-11T02:31:51,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/4277867e578c4f72b1d8c276fa5d6898 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/4277867e578c4f72b1d8c276fa5d6898 2024-11-11T02:31:51,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b78a867e876d4fd3914e3af416ea161f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b78a867e876d4fd3914e3af416ea161f 2024-11-11T02:31:51,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/315d90da515a4ed1b3a435c5ae853310 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/315d90da515a4ed1b3a435c5ae853310 2024-11-11T02:31:51,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/8ee3429674514e42b6b9e11dfc784f32 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/8ee3429674514e42b6b9e11dfc784f32 2024-11-11T02:31:51,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d6e65565d357402b9f979c1053c0885b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d6e65565d357402b9f979c1053c0885b 2024-11-11T02:31:51,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/133314cf2a814006917cd08700da733a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/133314cf2a814006917cd08700da733a 2024-11-11T02:31:51,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/cbf7c74d8d36444da7e11317c0e768b8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/cbf7c74d8d36444da7e11317c0e768b8 2024-11-11T02:31:51,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d7ed3abccfa6414abbe159065b1800a4 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/d7ed3abccfa6414abbe159065b1800a4 2024-11-11T02:31:51,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b5c3a96667344fd8a25c4d6d423ef493 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/b5c3a96667344fd8a25c4d6d423ef493 2024-11-11T02:31:51,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/e75bf71fc83e485985686708e0a8f9cc to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/e75bf71fc83e485985686708e0a8f9cc 2024-11-11T02:31:51,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f0103a3c5c9d41908367dfc2cffdcd25 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f0103a3c5c9d41908367dfc2cffdcd25 2024-11-11T02:31:51,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/76357a7eebb9470092d1866f9bf32827 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/76357a7eebb9470092d1866f9bf32827 2024-11-11T02:31:51,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f061d5f00a0443c9abcfcd44f79acf22 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/f061d5f00a0443c9abcfcd44f79acf22 2024-11-11T02:31:51,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/68da243be80f47f099ba165219e2bc47 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/68da243be80f47f099ba165219e2bc47 2024-11-11T02:31:51,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/114095b410394c2ab2a4fa3fef7c6f4d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/3e00b9c1fc1b458f83aa4dc0687023ce, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/190b8ba269544aa4ba54bc86857401f0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/c593a718b84a4a0c8e9dba1365b272eb, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/fcf5a484656548e4919d328d1cfda54f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cf0bf35df65f4d77a83d1da128136d6b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/fbe998d333bc4ce1babc9c788befe256, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/d3861670e01448bd875d5e8db6aac787, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cc838f1e165442d1baaf332db8738a76, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/0a9f033a3df244aeac256d33bb96f8e6, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/787882435a884ca49ca49f3c54a1a348, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/ccebabd1143e4cc8b1fa25f9f16bb15a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/1fa45d6fb3e440bb8ee52ca2ff0a2563, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/bdfe57ecd09f443b8a7f3a2c832e5e03, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/2c841996fafb4181a91aad43263d0cad, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/4fded1385294415992ef458a56b87add, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/497c01272d954b5caa59501a0970bd7a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/86b4f099523748369a264202f317e71b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/d6c2b26e089a4aaaaeb247c499b8ede8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/8f96932906334341b97df3e87543e9cc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cbb5d2cb23ce4ae1936128b771cd8704, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/c76c0cd78cb54d83acabf5035b42e599, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/942cef60a5384ebca30f130837c7ddb0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/1f6e71fcc9b54404a2b268573bf945a9, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/bdd33af4341b4cfcbb22ee8b4420bb6b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/71b882058d5044f0b5af35edb1e29ed8] to archive 2024-11-11T02:31:51,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:31:51,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/114095b410394c2ab2a4fa3fef7c6f4d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/114095b410394c2ab2a4fa3fef7c6f4d 2024-11-11T02:31:51,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/3e00b9c1fc1b458f83aa4dc0687023ce to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/3e00b9c1fc1b458f83aa4dc0687023ce 2024-11-11T02:31:51,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/190b8ba269544aa4ba54bc86857401f0 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/190b8ba269544aa4ba54bc86857401f0 2024-11-11T02:31:51,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/c593a718b84a4a0c8e9dba1365b272eb to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/c593a718b84a4a0c8e9dba1365b272eb 2024-11-11T02:31:51,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/fcf5a484656548e4919d328d1cfda54f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/fcf5a484656548e4919d328d1cfda54f 2024-11-11T02:31:51,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cf0bf35df65f4d77a83d1da128136d6b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cf0bf35df65f4d77a83d1da128136d6b 2024-11-11T02:31:51,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/fbe998d333bc4ce1babc9c788befe256 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/fbe998d333bc4ce1babc9c788befe256 2024-11-11T02:31:51,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/d3861670e01448bd875d5e8db6aac787 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/d3861670e01448bd875d5e8db6aac787 2024-11-11T02:31:51,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cc838f1e165442d1baaf332db8738a76 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cc838f1e165442d1baaf332db8738a76 2024-11-11T02:31:51,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/0a9f033a3df244aeac256d33bb96f8e6 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/0a9f033a3df244aeac256d33bb96f8e6 2024-11-11T02:31:51,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/787882435a884ca49ca49f3c54a1a348 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/787882435a884ca49ca49f3c54a1a348 2024-11-11T02:31:51,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/ccebabd1143e4cc8b1fa25f9f16bb15a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/ccebabd1143e4cc8b1fa25f9f16bb15a 2024-11-11T02:31:51,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/1fa45d6fb3e440bb8ee52ca2ff0a2563 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/1fa45d6fb3e440bb8ee52ca2ff0a2563 2024-11-11T02:31:51,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/bdfe57ecd09f443b8a7f3a2c832e5e03 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/bdfe57ecd09f443b8a7f3a2c832e5e03 2024-11-11T02:31:51,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/2c841996fafb4181a91aad43263d0cad to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/2c841996fafb4181a91aad43263d0cad 2024-11-11T02:31:51,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/4fded1385294415992ef458a56b87add to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/4fded1385294415992ef458a56b87add 2024-11-11T02:31:51,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/497c01272d954b5caa59501a0970bd7a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/497c01272d954b5caa59501a0970bd7a 2024-11-11T02:31:51,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/86b4f099523748369a264202f317e71b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/86b4f099523748369a264202f317e71b 2024-11-11T02:31:51,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/d6c2b26e089a4aaaaeb247c499b8ede8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/d6c2b26e089a4aaaaeb247c499b8ede8 2024-11-11T02:31:51,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/8f96932906334341b97df3e87543e9cc to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/8f96932906334341b97df3e87543e9cc 2024-11-11T02:31:51,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cbb5d2cb23ce4ae1936128b771cd8704 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/cbb5d2cb23ce4ae1936128b771cd8704 2024-11-11T02:31:51,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/c76c0cd78cb54d83acabf5035b42e599 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/c76c0cd78cb54d83acabf5035b42e599 2024-11-11T02:31:51,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/942cef60a5384ebca30f130837c7ddb0 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/942cef60a5384ebca30f130837c7ddb0 2024-11-11T02:31:51,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/1f6e71fcc9b54404a2b268573bf945a9 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/1f6e71fcc9b54404a2b268573bf945a9 2024-11-11T02:31:51,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/bdd33af4341b4cfcbb22ee8b4420bb6b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/bdd33af4341b4cfcbb22ee8b4420bb6b 2024-11-11T02:31:51,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/71b882058d5044f0b5af35edb1e29ed8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/71b882058d5044f0b5af35edb1e29ed8 2024-11-11T02:31:51,718 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/recovered.edits/426.seqid, newMaxSeqId=426, maxSeqId=4 2024-11-11T02:31:51,719 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. 2024-11-11T02:31:51,719 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for e33fc31e962e33bc26f57b81ef4a0f99: Waiting for close lock at 1731292309923Running coprocessor pre-close hooks at 1731292309923Disabling compacts and flushes for region at 1731292309923Disabling writes for close at 1731292309924 (+1 ms)Obtaining lock to block concurrent updates at 1731292309924Preparing flush snapshotting stores in e33fc31e962e33bc26f57b81ef4a0f99 at 1731292309924Finished memstore snapshotting TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731292309925 (+1 ms)Flushing stores of TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99. at 1731292309926 (+1 ms)Flushing e33fc31e962e33bc26f57b81ef4a0f99/A: creating writer at 1731292309926Flushing mob file e33fc31e962e33bc26f57b81ef4a0f99/A: appending metadata at 1731292309935 (+9 ms)Flushing mob file e33fc31e962e33bc26f57b81ef4a0f99/A: closing flushed file at 1731292309935Flushing e33fc31e962e33bc26f57b81ef4a0f99/A: appending metadata at 1731292310350 (+415 ms)Flushing e33fc31e962e33bc26f57b81ef4a0f99/A: closing flushed file at 1731292310350Flushing e33fc31e962e33bc26f57b81ef4a0f99/B: creating writer at 1731292310767 (+417 ms)Flushing e33fc31e962e33bc26f57b81ef4a0f99/B: appending metadata at 1731292310773 (+6 ms)Flushing e33fc31e962e33bc26f57b81ef4a0f99/B: closing flushed file at 1731292310773Flushing e33fc31e962e33bc26f57b81ef4a0f99/C: creating writer at 1731292311189 (+416 ms)Flushing e33fc31e962e33bc26f57b81ef4a0f99/C: appending metadata at 1731292311193 (+4 ms)Flushing e33fc31e962e33bc26f57b81ef4a0f99/C: closing flushed file at 1731292311193Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3948c0a1: reopening flushed file at 1731292311608 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cef9ea0: reopening flushed file at 1731292311614 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@584c464a: reopening flushed file at 1731292311619 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e33fc31e962e33bc26f57b81ef4a0f99 in 1701ms, sequenceid=423, compaction requested=true at 1731292311625 (+6 ms)Writing region close event to WAL at 1731292311715 (+90 ms)Running coprocessor post-close hooks at 1731292311718 (+3 ms)Closed at 1731292311719 (+1 ms) 2024-11-11T02:31:51,721 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,721 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=e33fc31e962e33bc26f57b81ef4a0f99, regionState=CLOSED 2024-11-11T02:31:51,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:31:51,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-11T02:31:51,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure e33fc31e962e33bc26f57b81ef4a0f99, server=370bc2ade342,38037,1731292247443 in 1.9530 sec 2024-11-11T02:31:51,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-11T02:31:51,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e33fc31e962e33bc26f57b81ef4a0f99, UNASSIGN in 1.9590 sec 2024-11-11T02:31:51,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-11T02:31:51,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9630 sec 2024-11-11T02:31:51,730 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292311730"}]},"ts":"1731292311730"} 2024-11-11T02:31:51,731 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T02:31:51,731 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T02:31:51,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9770 sec 2024-11-11T02:31:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T02:31:51,905 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:51,906 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T02:31:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:51,908 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T02:31:51,909 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:51,911 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,916 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/recovered.edits] 2024-11-11T02:31:51,921 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/1c46539c38ca47f198edd8e6317b276d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/1c46539c38ca47f198edd8e6317b276d 2024-11-11T02:31:51,923 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ad6afb8d1b584ed885de7400005822c6 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/ad6afb8d1b584ed885de7400005822c6 2024-11-11T02:31:51,925 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f783dcf146ab43eba4e742a1f230a32c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/A/f783dcf146ab43eba4e742a1f230a32c 2024-11-11T02:31:51,927 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/0a76dc5f52ce48e3a7ebc487236b990c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/0a76dc5f52ce48e3a7ebc487236b990c 2024-11-11T02:31:51,928 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/3f4e8821cbb34da4bb7306ac21b09960 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/3f4e8821cbb34da4bb7306ac21b09960 2024-11-11T02:31:51,929 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/da34b0f1d61e452c9d56340c56606ada to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/B/da34b0f1d61e452c9d56340c56606ada 2024-11-11T02:31:51,931 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/9157898565184eaca9e6488465c95b5c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/9157898565184eaca9e6488465c95b5c 2024-11-11T02:31:51,931 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/e531d186af624669857b18219ef46fe9 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/e531d186af624669857b18219ef46fe9 2024-11-11T02:31:51,932 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/e5719440ee81498d809010838cb7274c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/C/e5719440ee81498d809010838cb7274c 2024-11-11T02:31:51,935 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/recovered.edits/426.seqid to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99/recovered.edits/426.seqid 2024-11-11T02:31:51,935 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,936 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T02:31:51,936 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T02:31:51,937 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T02:31:51,940 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111100d8c882c6b84e83ae77895ad31abf2a_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111100d8c882c6b84e83ae77895ad31abf2a_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,941 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111102f7a1ffeed9415fa100c4bc59abb74f_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111102f7a1ffeed9415fa100c4bc59abb74f_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,943 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111033912d764f64d789dcc2ed7633fb25f_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111033912d764f64d789dcc2ed7633fb25f_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,944 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111080bffd3e7ce4d97956c00e47abeeace_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111080bffd3e7ce4d97956c00e47abeeace_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,945 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111137b2f3779ba74aed9b110c13c6f54ee8_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111137b2f3779ba74aed9b110c13c6f54ee8_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,946 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146b858ea7398454cb0e45b87b2a2ff1c_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146b858ea7398454cb0e45b87b2a2ff1c_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,947 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114ad67d2314df4168932ee9c123542d8e_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114ad67d2314df4168932ee9c123542d8e_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,948 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114fd598d1d6454740862abc1a0c3affdd_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114fd598d1d6454740862abc1a0c3affdd_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,949 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111154c4a7fe27304f358f5ff215795225be_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111154c4a7fe27304f358f5ff215795225be_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,950 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111165c67271b1584460a5f8f138c0f86689_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111165c67271b1584460a5f8f138c0f86689_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,951 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111737e338727bf4b2c91973ee800609acf_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111737e338727bf4b2c91973ee800609acf_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,952 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111173f6e914662c487e9383a58b1307210f_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111173f6e914662c487e9383a58b1307210f_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,952 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118059c6f2f7144fe59b99678e717ff29f_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118059c6f2f7144fe59b99678e717ff29f_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,953 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ac40d7dfc64d486680b73e9c9c4a89cb_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ac40d7dfc64d486680b73e9c9c4a89cb_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,954 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b2e837c221b646a6afd8b0f8a5a56c69_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b2e837c221b646a6afd8b0f8a5a56c69_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,955 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bbd5b0220393411ea61d69a9a4ba61c6_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bbd5b0220393411ea61d69a9a4ba61c6_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,956 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bec49c213f674560b1c0d94f08ab7078_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bec49c213f674560b1c0d94f08ab7078_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,957 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cfb9430d530c4b84897bd8fbfa0bc06c_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cfb9430d530c4b84897bd8fbfa0bc06c_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,958 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d36f2875794d49a985b8811c711e6799_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d36f2875794d49a985b8811c711e6799_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,959 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5e6665a946848b68389fe744b7d85eb_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5e6665a946848b68389fe744b7d85eb_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,960 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d70b1629518b49f0ae5a777b937eff99_e33fc31e962e33bc26f57b81ef4a0f99 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d70b1629518b49f0ae5a777b937eff99_e33fc31e962e33bc26f57b81ef4a0f99 2024-11-11T02:31:51,960 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T02:31:51,962 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:51,964 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T02:31:51,966 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T02:31:51,968 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:51,968 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T02:31:51,968 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731292311968"}]},"ts":"9223372036854775807"} 2024-11-11T02:31:51,970 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T02:31:51,970 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e33fc31e962e33bc26f57b81ef4a0f99, NAME => 'TestAcidGuarantees,,1731292276752.e33fc31e962e33bc26f57b81ef4a0f99.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T02:31:51,970 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T02:31:51,971 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731292311970"}]},"ts":"9223372036854775807"} 2024-11-11T02:31:51,972 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T02:31:51,973 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:51,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-11-11T02:31:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T02:31:52,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T02:31:52,016 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:52,025 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=744 (was 973), ProcessCount=11 (was 11), AvailableMemoryMB=5676 (was 5257) - AvailableMemoryMB LEAK? - 2024-11-11T02:31:52,034 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=744, ProcessCount=11, AvailableMemoryMB=5675 2024-11-11T02:31:52,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T02:31:52,035 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:31:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T02:31:52,037 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T02:31:52,037 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:52,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-11T02:31:52,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-11T02:31:52,038 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T02:31:52,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742055_1231 (size=1061) 2024-11-11T02:31:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-11T02:31:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-11T02:31:52,448 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe 2024-11-11T02:31:52,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742056_1232 (size=53) 2024-11-11T02:31:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-11T02:31:52,861 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:31:52,861 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3f251ac37bffb103f032beac897bfe5d, disabling compactions & flushes 2024-11-11T02:31:52,861 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:52,861 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:52,861 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. after waiting 0 ms 2024-11-11T02:31:52,861 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:52,861 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:52,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3f251ac37bffb103f032beac897bfe5d: Waiting for close lock at 1731292312861Disabling compacts and flushes for region at 1731292312861Disabling writes for close at 1731292312861Writing region close event to WAL at 1731292312861Closed at 1731292312861 2024-11-11T02:31:52,864 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T02:31:52,865 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731292312864"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731292312864"}]},"ts":"1731292312864"} 2024-11-11T02:31:52,868 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T02:31:52,870 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T02:31:52,871 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292312870"}]},"ts":"1731292312870"} 2024-11-11T02:31:52,873 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T02:31:52,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f251ac37bffb103f032beac897bfe5d, ASSIGN}] 2024-11-11T02:31:52,874 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f251ac37bffb103f032beac897bfe5d, ASSIGN 2024-11-11T02:31:52,875 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f251ac37bffb103f032beac897bfe5d, ASSIGN; state=OFFLINE, location=370bc2ade342,38037,1731292247443; forceNewPlan=false, retain=false 2024-11-11T02:31:53,026 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=3f251ac37bffb103f032beac897bfe5d, regionState=OPENING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:31:53,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f251ac37bffb103f032beac897bfe5d, ASSIGN because future has completed 2024-11-11T02:31:53,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:31:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-11T02:31:53,189 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:53,189 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:31:53,190 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:53,190 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:31:53,190 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:53,191 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:53,194 INFO [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:53,197 INFO [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:31:53,197 INFO [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f251ac37bffb103f032beac897bfe5d columnFamilyName A 2024-11-11T02:31:53,197 DEBUG [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:53,198 INFO [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] regionserver.HStore(327): Store=3f251ac37bffb103f032beac897bfe5d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:31:53,198 INFO [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:53,201 INFO [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:31:53,201 INFO [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f251ac37bffb103f032beac897bfe5d columnFamilyName B 2024-11-11T02:31:53,201 DEBUG [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:53,202 INFO [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] regionserver.HStore(327): Store=3f251ac37bffb103f032beac897bfe5d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:31:53,203 INFO [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:53,205 INFO [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:31:53,205 INFO [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f251ac37bffb103f032beac897bfe5d columnFamilyName C 2024-11-11T02:31:53,205 DEBUG [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:31:53,205 INFO [StoreOpener-3f251ac37bffb103f032beac897bfe5d-1 {}] regionserver.HStore(327): Store=3f251ac37bffb103f032beac897bfe5d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:31:53,206 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:53,206 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:53,206 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:53,207 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:53,208 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:53,208 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:53,209 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:31:53,210 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:53,212 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:31:53,213 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened 3f251ac37bffb103f032beac897bfe5d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70234363, jitterRate=0.046573564410209656}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:31:53,213 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:53,214 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for 3f251ac37bffb103f032beac897bfe5d: Running coprocessor pre-open hook at 1731292313191Writing region info on filesystem at 1731292313191Initializing all the Stores at 1731292313193 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292313193Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292313193Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292313193Cleaning up temporary data from old regions at 1731292313208 (+15 ms)Running coprocessor post-open hooks at 1731292313213 (+5 ms)Region opened successfully at 1731292313213 2024-11-11T02:31:53,214 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., pid=67, masterSystemTime=1731292313184 2024-11-11T02:31:53,216 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:53,217 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:53,217 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=3f251ac37bffb103f032beac897bfe5d, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:31:53,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:31:53,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-11T02:31:53,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 in 190 msec 2024-11-11T02:31:53,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-11T02:31:53,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f251ac37bffb103f032beac897bfe5d, ASSIGN in 349 msec 2024-11-11T02:31:53,225 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T02:31:53,226 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292313225"}]},"ts":"1731292313225"} 2024-11-11T02:31:53,227 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T02:31:53,228 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T02:31:53,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1930 sec 2024-11-11T02:31:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-11T02:31:54,186 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:54,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:54,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:54,188 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7832b28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:54,190 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a1e1c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,190 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:54,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:54,193 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T02:31:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 370bc2ade342,42785,1731292246873 2024-11-11T02:31:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@159f1899 2024-11-11T02:31:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T02:31:54,194 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T02:31:54,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:54,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:54,196 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323e29e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:54,198 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aae2315, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,199 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,199 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:54,199 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:54,200 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683cacf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:54,201 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c7d198, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,202 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:54,202 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:54,203 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a79666a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:54,204 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a0f6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,205 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:54,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:54,206 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c48698b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:54,207 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e5595df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,209 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:54,209 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:54,209 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:54,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:54,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:54,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79e747a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:54,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:54,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:54,211 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf1277c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:54,212 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:54,213 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b58f638, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:54,215 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:54,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23fd41a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:54,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:54,217 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:54,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:54,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:54,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@667b3002, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:54,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:54,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:54,218 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:54,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c065b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,220 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:54,220 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:54,221 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78019144, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:54,222 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:54,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e9d7c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,223 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:54,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:54,224 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@350b8260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:54,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:54,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:54,225 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24af746a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,227 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:31:54,227 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:31:54,228 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:31:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:31:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:31:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0d3437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:31:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:31:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:31:54,229 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:31:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c259133, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:31:54,230 DEBUG [Thread-1104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:54,230 DEBUG [Thread-1089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:54,231 DEBUG [Thread-1093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:54,231 DEBUG [Thread-1098 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:54,231 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:54,232 DEBUG [Thread-1093 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:54,232 DEBUG [Thread-1102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:54,232 DEBUG [Thread-1093 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:54,232 DEBUG [Thread-1096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:54,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:54,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:54,232 DEBUG [Thread-1085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-11T02:31:54,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:54,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:54,233 DEBUG [Thread-1100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:54,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T02:31:54,234 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:54,234 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:54,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:31:54,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:54,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:31:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:31:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:31:54,237 DEBUG [Thread-1087 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:54,237 DEBUG [Thread-1098 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:54,237 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:54,237 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:54,237 DEBUG [Thread-1098 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:54,237 DEBUG [Thread-1096 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:54,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:54,237 DEBUG [Thread-1096 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:54,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:54,239 DEBUG [Thread-1091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:31:54,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:54,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:54,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:31:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:31:54,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:31:54,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:31:54,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:31:54,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:31:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:31:54,242 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:31:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:31:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:54,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:31:54,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:31:54,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:54,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:31:54,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:54,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:31:54,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:54,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/3dd1df037fe947fea24232ac5d85788e is 50, key is test_row_0/A:col10/1731292314248/Put/seqid=0 2024-11-11T02:31:54,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742057_1233 (size=12001) 2024-11-11T02:31:54,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292374279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292374280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292374280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292314278, max timestamp=1731292314285Num uniques 859; 2024-11-11T02:31:54,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:31:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292314278, max timestamp=1731292314285Num uniques 859; 2024-11-11T02:31:54,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:31:54,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292314278, max timestamp=1731292314285Num uniques 859; 2024-11-11T02:31:54,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:31:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T02:31:54,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292374345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292374349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T02:31:54,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:54,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:31:54,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:54,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:54,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:54,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292374395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292374395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292374395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292374455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292374455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T02:31:54,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:54,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:31:54,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:54,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:54,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T02:31:54,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292374606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292374606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292374606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292374665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292374666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/3dd1df037fe947fea24232ac5d85788e 2024-11-11T02:31:54,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T02:31:54,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:54,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:31:54,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:54,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:54,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:54,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/99c702fcf85045a686b2ced9372d479c is 50, key is test_row_0/B:col10/1731292314248/Put/seqid=0 2024-11-11T02:31:54,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742058_1234 (size=12001) 2024-11-11T02:31:54,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T02:31:54,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:54,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:31:54,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:54,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:54,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T02:31:54,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292374916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292374916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292374916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292374975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292374975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:54,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:54,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:55,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T02:31:55,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:55,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:31:55,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:55,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:55,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:55,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/99c702fcf85045a686b2ced9372d479c 2024-11-11T02:31:55,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/29acc1e31b4b4f43ac61b95e1d8ec855 is 50, key is test_row_0/C:col10/1731292314248/Put/seqid=0 2024-11-11T02:31:55,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742059_1235 (size=12001) 2024-11-11T02:31:55,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/29acc1e31b4b4f43ac61b95e1d8ec855 2024-11-11T02:31:55,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/3dd1df037fe947fea24232ac5d85788e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/3dd1df037fe947fea24232ac5d85788e 2024-11-11T02:31:55,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/3dd1df037fe947fea24232ac5d85788e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T02:31:55,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/99c702fcf85045a686b2ced9372d479c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/99c702fcf85045a686b2ced9372d479c 2024-11-11T02:31:55,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T02:31:55,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:55,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:31:55,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:55,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:55,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:55,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/99c702fcf85045a686b2ced9372d479c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T02:31:55,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/29acc1e31b4b4f43ac61b95e1d8ec855 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/29acc1e31b4b4f43ac61b95e1d8ec855 2024-11-11T02:31:55,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/29acc1e31b4b4f43ac61b95e1d8ec855, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T02:31:55,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 3f251ac37bffb103f032beac897bfe5d in 921ms, sequenceid=13, compaction requested=false 2024-11-11T02:31:55,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:31:55,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T02:31:55,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:55,311 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-11-11T02:31:55,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:31:55,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:55,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:31:55,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:55,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:31:55,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:31:55,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/06fe342b8db142beb227031e8fb7baa4 is 50, key is test_row_0/A:col10/1731292314344/Put/seqid=0 2024-11-11T02:31:55,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742060_1236 (size=12001) 2024-11-11T02:31:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T02:31:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:55,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:31:55,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292375439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:55,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292375442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292375443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:55,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292375486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292375486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:55,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:55,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292375555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292375555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:55,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292375555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:55,720 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/06fe342b8db142beb227031e8fb7baa4 2024-11-11T02:31:55,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/b351be31e0e442d8b267fcc14db4daa8 is 50, key is test_row_0/B:col10/1731292314344/Put/seqid=0 2024-11-11T02:31:55,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742061_1237 (size=12001) 2024-11-11T02:31:55,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292375765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:55,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:55,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292375765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:55,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:55,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292375765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292376076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292376076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292376076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,137 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/b351be31e0e442d8b267fcc14db4daa8 2024-11-11T02:31:56,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/fc480f9fb4164776a84388432cbf12b1 is 50, key is test_row_0/C:col10/1731292314344/Put/seqid=0 2024-11-11T02:31:56,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742062_1238 (size=12001) 2024-11-11T02:31:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T02:31:56,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292376495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:56,498 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in 3f251ac37bffb103f032beac897bfe5d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292376505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:56,509 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in 3f251ac37bffb103f032beac897bfe5d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:56,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,552 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/fc480f9fb4164776a84388432cbf12b1 2024-11-11T02:31:56,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/06fe342b8db142beb227031e8fb7baa4 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/06fe342b8db142beb227031e8fb7baa4 2024-11-11T02:31:56,560 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/06fe342b8db142beb227031e8fb7baa4, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T02:31:56,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/b351be31e0e442d8b267fcc14db4daa8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/b351be31e0e442d8b267fcc14db4daa8 2024-11-11T02:31:56,565 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/b351be31e0e442d8b267fcc14db4daa8, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T02:31:56,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/fc480f9fb4164776a84388432cbf12b1 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/fc480f9fb4164776a84388432cbf12b1 2024-11-11T02:31:56,570 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/fc480f9fb4164776a84388432cbf12b1, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T02:31:56,571 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for 3f251ac37bffb103f032beac897bfe5d in 1260ms, sequenceid=48, compaction requested=false 2024-11-11T02:31:56,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:31:56,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:56,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-11T02:31:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-11T02:31:56,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-11T02:31:56,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3360 sec 2024-11-11T02:31:56,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.3440 sec 2024-11-11T02:31:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:56,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T02:31:56,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:31:56,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:56,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:31:56,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:56,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:31:56,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:56,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/50e784ef72074e739a0e73b0f6f1fddf is 50, key is test_row_0/A:col10/1731292316587/Put/seqid=0 2024-11-11T02:31:56,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742063_1239 (size=16681) 2024-11-11T02:31:56,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292376614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:56,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:56,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292376615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:56,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292376617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292376726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292376726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292376726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292376935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292376935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:56,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292376935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:56,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/50e784ef72074e739a0e73b0f6f1fddf 2024-11-11T02:31:57,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/3cccee7e35244fe3879399f20ffd02c6 is 50, key is test_row_0/B:col10/1731292316587/Put/seqid=0 2024-11-11T02:31:57,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742064_1240 (size=12001) 2024-11-11T02:31:57,242 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T02:31:57,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292377246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:57,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292377246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:57,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292377246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:57,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/3cccee7e35244fe3879399f20ffd02c6 2024-11-11T02:31:57,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/78684e78c1b740938ff50bd6d1f709df is 50, key is test_row_0/C:col10/1731292316587/Put/seqid=0 2024-11-11T02:31:57,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742065_1241 (size=12001) 2024-11-11T02:31:57,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:57,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292377756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:57,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292377765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:57,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292377765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:57,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/78684e78c1b740938ff50bd6d1f709df 2024-11-11T02:31:57,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/50e784ef72074e739a0e73b0f6f1fddf as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/50e784ef72074e739a0e73b0f6f1fddf 2024-11-11T02:31:57,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/50e784ef72074e739a0e73b0f6f1fddf, entries=250, sequenceid=62, filesize=16.3 K 2024-11-11T02:31:57,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/3cccee7e35244fe3879399f20ffd02c6 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/3cccee7e35244fe3879399f20ffd02c6 2024-11-11T02:31:57,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/3cccee7e35244fe3879399f20ffd02c6, entries=150, sequenceid=62, filesize=11.7 K 2024-11-11T02:31:57,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/78684e78c1b740938ff50bd6d1f709df as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/78684e78c1b740938ff50bd6d1f709df 2024-11-11T02:31:57,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/78684e78c1b740938ff50bd6d1f709df, entries=150, sequenceid=62, filesize=11.7 K 2024-11-11T02:31:57,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3f251ac37bffb103f032beac897bfe5d in 1252ms, sequenceid=62, compaction requested=true 2024-11-11T02:31:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:31:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:31:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:57,841 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:31:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:57,841 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:31:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:57,842 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:57,842 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:57,842 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/B is initiating minor compaction (all files) 2024-11-11T02:31:57,842 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/A is initiating minor compaction (all files) 2024-11-11T02:31:57,842 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/A in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:57,842 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/B in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:57,842 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/3dd1df037fe947fea24232ac5d85788e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/06fe342b8db142beb227031e8fb7baa4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/50e784ef72074e739a0e73b0f6f1fddf] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=39.7 K 2024-11-11T02:31:57,842 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/99c702fcf85045a686b2ced9372d479c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/b351be31e0e442d8b267fcc14db4daa8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/3cccee7e35244fe3879399f20ffd02c6] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=35.2 K 2024-11-11T02:31:57,842 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dd1df037fe947fea24232ac5d85788e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731292314246 2024-11-11T02:31:57,843 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 99c702fcf85045a686b2ced9372d479c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731292314246 2024-11-11T02:31:57,843 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06fe342b8db142beb227031e8fb7baa4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731292314342 2024-11-11T02:31:57,843 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting b351be31e0e442d8b267fcc14db4daa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731292314342 2024-11-11T02:31:57,843 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50e784ef72074e739a0e73b0f6f1fddf, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731292315429 2024-11-11T02:31:57,843 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cccee7e35244fe3879399f20ffd02c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731292315441 2024-11-11T02:31:57,851 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:57,852 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/98655c671ebf4ce0811cf027f522bdce is 50, key is test_row_0/B:col10/1731292316587/Put/seqid=0 2024-11-11T02:31:57,855 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#A#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:57,855 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/4df0a5f8291449c3a87e825d017b374e is 50, key is test_row_0/A:col10/1731292316587/Put/seqid=0 2024-11-11T02:31:57,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742066_1242 (size=12104) 2024-11-11T02:31:57,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742067_1243 (size=12104) 2024-11-11T02:31:57,877 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/4df0a5f8291449c3a87e825d017b374e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/4df0a5f8291449c3a87e825d017b374e 2024-11-11T02:31:57,882 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/A of 3f251ac37bffb103f032beac897bfe5d into 4df0a5f8291449c3a87e825d017b374e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:57,882 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:31:57,882 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/A, priority=13, startTime=1731292317841; duration=0sec 2024-11-11T02:31:57,882 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:31:57,882 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:A 2024-11-11T02:31:57,882 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:31:57,883 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:31:57,883 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/C is initiating minor compaction (all files) 2024-11-11T02:31:57,883 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/C in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:57,883 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/29acc1e31b4b4f43ac61b95e1d8ec855, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/fc480f9fb4164776a84388432cbf12b1, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/78684e78c1b740938ff50bd6d1f709df] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=35.2 K 2024-11-11T02:31:57,884 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29acc1e31b4b4f43ac61b95e1d8ec855, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731292314246 2024-11-11T02:31:57,884 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc480f9fb4164776a84388432cbf12b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731292314342 2024-11-11T02:31:57,884 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78684e78c1b740938ff50bd6d1f709df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731292315441 2024-11-11T02:31:57,891 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#C#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:31:57,892 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/47c719b86c3f4df3b6a01e8d5095ce23 is 50, key is test_row_0/C:col10/1731292316587/Put/seqid=0 2024-11-11T02:31:57,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742068_1244 (size=12104) 2024-11-11T02:31:58,268 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/98655c671ebf4ce0811cf027f522bdce as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/98655c671ebf4ce0811cf027f522bdce 2024-11-11T02:31:58,272 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/B of 3f251ac37bffb103f032beac897bfe5d into 98655c671ebf4ce0811cf027f522bdce(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:58,272 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:31:58,272 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/B, priority=13, startTime=1731292317841; duration=0sec 2024-11-11T02:31:58,273 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:58,273 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:B 2024-11-11T02:31:58,311 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/47c719b86c3f4df3b6a01e8d5095ce23 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/47c719b86c3f4df3b6a01e8d5095ce23 2024-11-11T02:31:58,316 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/C of 3f251ac37bffb103f032beac897bfe5d into 47c719b86c3f4df3b6a01e8d5095ce23(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:31:58,316 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:31:58,316 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/C, priority=13, startTime=1731292317841; duration=0sec 2024-11-11T02:31:58,316 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:31:58,316 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:C 2024-11-11T02:31:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T02:31:58,395 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:31:58,396 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:31:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-11T02:31:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T02:31:58,398 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:31:58,398 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:31:58,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:31:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T02:31:58,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:58,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T02:31:58,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:31:58,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:58,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:31:58,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:58,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:31:58,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:58,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/93bce23e0e044e5491b91ef25d9d9158 is 50, key is test_row_0/A:col10/1731292318508/Put/seqid=0 2024-11-11T02:31:58,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742069_1245 (size=12001) 2024-11-11T02:31:58,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292378539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:58,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:58,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292378542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:58,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T02:31:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:31:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:58,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:58,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292378645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:58,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:58,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292378655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:58,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:58,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T02:31:58,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:58,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:31:58,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:58,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:58,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T02:31:58,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292378765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:58,769 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in 3f251ac37bffb103f032beac897bfe5d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:58,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292378776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:58,778 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in 3f251ac37bffb103f032beac897bfe5d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:58,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292378785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:58,786 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in 3f251ac37bffb103f032beac897bfe5d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:31:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:58,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T02:31:58,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:58,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:31:58,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:58,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:58,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:58,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292378856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:58,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292378866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:58,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:58,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:58,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/93bce23e0e044e5491b91ef25d9d9158 2024-11-11T02:31:58,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/17689d5f56fc413ea0fe1549fac6a05b is 50, key is test_row_0/B:col10/1731292318508/Put/seqid=0 2024-11-11T02:31:58,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742070_1246 (size=12001) 2024-11-11T02:31:58,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/17689d5f56fc413ea0fe1549fac6a05b 2024-11-11T02:31:58,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/c583220ac66c48e88ab73795096445e2 is 50, key is test_row_0/C:col10/1731292318508/Put/seqid=0 2024-11-11T02:31:58,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742071_1247 (size=12001) 2024-11-11T02:31:59,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T02:31:59,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:31:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:59,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:59,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T02:31:59,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T02:31:59,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:59,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:31:59,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:59,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:59,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:59,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:59,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292379165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:59,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:59,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292379175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:59,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T02:31:59,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:59,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:31:59,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:59,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:59,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:31:59,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/c583220ac66c48e88ab73795096445e2 2024-11-11T02:31:59,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/93bce23e0e044e5491b91ef25d9d9158 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/93bce23e0e044e5491b91ef25d9d9158 2024-11-11T02:31:59,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/93bce23e0e044e5491b91ef25d9d9158, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T02:31:59,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/17689d5f56fc413ea0fe1549fac6a05b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/17689d5f56fc413ea0fe1549fac6a05b 2024-11-11T02:31:59,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/17689d5f56fc413ea0fe1549fac6a05b, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T02:31:59,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/c583220ac66c48e88ab73795096445e2 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/c583220ac66c48e88ab73795096445e2 2024-11-11T02:31:59,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/c583220ac66c48e88ab73795096445e2, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T02:31:59,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 3f251ac37bffb103f032beac897bfe5d in 876ms, sequenceid=84, compaction requested=false 2024-11-11T02:31:59,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:31:59,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T02:31:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:31:59,470 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T02:31:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:31:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:31:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:31:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:31:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/48aacb08a91d4f1aafb560f21b905ce1 is 50, key is test_row_0/A:col10/1731292318538/Put/seqid=0 2024-11-11T02:31:59,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742072_1248 (size=12001) 2024-11-11T02:31:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T02:31:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:31:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:31:59,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292379716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:59,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:59,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:59,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:59,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292379719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:59,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:59,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:59,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:59,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292379825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:59,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:31:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292379825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:31:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:31:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:31:59,880 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/48aacb08a91d4f1aafb560f21b905ce1 2024-11-11T02:31:59,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/489b4e7869474f5dbd1ed8b66371ecdb is 50, key is test_row_0/B:col10/1731292318538/Put/seqid=0 2024-11-11T02:31:59,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742073_1249 (size=12001) 2024-11-11T02:31:59,893 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/489b4e7869474f5dbd1ed8b66371ecdb 2024-11-11T02:31:59,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/8d2826ea829f47c18129a27fdf749bb4 is 50, key is test_row_0/C:col10/1731292318538/Put/seqid=0 2024-11-11T02:31:59,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742074_1250 (size=12001) 2024-11-11T02:32:00,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:00,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292380035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292380035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:00,304 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/8d2826ea829f47c18129a27fdf749bb4 2024-11-11T02:32:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/48aacb08a91d4f1aafb560f21b905ce1 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/48aacb08a91d4f1aafb560f21b905ce1 2024-11-11T02:32:00,313 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/48aacb08a91d4f1aafb560f21b905ce1, entries=150, sequenceid=97, filesize=11.7 K 2024-11-11T02:32:00,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/489b4e7869474f5dbd1ed8b66371ecdb as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/489b4e7869474f5dbd1ed8b66371ecdb 2024-11-11T02:32:00,318 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/489b4e7869474f5dbd1ed8b66371ecdb, entries=150, sequenceid=97, filesize=11.7 K 2024-11-11T02:32:00,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/8d2826ea829f47c18129a27fdf749bb4 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8d2826ea829f47c18129a27fdf749bb4 2024-11-11T02:32:00,322 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8d2826ea829f47c18129a27fdf749bb4, entries=150, sequenceid=97, filesize=11.7 K 2024-11-11T02:32:00,323 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 3f251ac37bffb103f032beac897bfe5d in 853ms, sequenceid=97, compaction requested=true 2024-11-11T02:32:00,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:00,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:00,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-11T02:32:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-11T02:32:00,326 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-11T02:32:00,326 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9260 sec 2024-11-11T02:32:00,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9310 sec 2024-11-11T02:32:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:00,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T02:32:00,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:32:00,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:00,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:32:00,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:00,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:32:00,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:00,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/dc426f62c8034bdba97bdafc08e4a05a is 50, key is test_row_0/A:col10/1731292320348/Put/seqid=0 2024-11-11T02:32:00,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742075_1251 (size=14341) 2024-11-11T02:32:00,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292380370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:00,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292380373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:00,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292380486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292380485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T02:32:00,545 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:00,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-11T02:32:00,548 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T02:32:00,548 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:00,548 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T02:32:00,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292380695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:00,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292380695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:00,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T02:32:00,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:00,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:00,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:00,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:00,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:00,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/dc426f62c8034bdba97bdafc08e4a05a 2024-11-11T02:32:00,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/7275f8876a69447087beafef15a73318 is 50, key is test_row_0/B:col10/1731292320348/Put/seqid=0 2024-11-11T02:32:00,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742076_1252 (size=12001) 2024-11-11T02:32:00,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292380785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:00,789 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in 3f251ac37bffb103f032beac897bfe5d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:32:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:00,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292380806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:00,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292380806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:00,809 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in 3f251ac37bffb103f032beac897bfe5d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:32:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,809 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in 3f251ac37bffb103f032beac897bfe5d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:32:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:00,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:00,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:00,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T02:32:00,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:00,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:00,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:00,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:00,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T02:32:01,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T02:32:01,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:01,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:01,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:01,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:01,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:01,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292381019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:01,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292381019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:01,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T02:32:01,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:01,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:01,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:01,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:01,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T02:32:01,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/7275f8876a69447087beafef15a73318 2024-11-11T02:32:01,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/5eaf09d21f9d412694b4bdda17d2163f is 50, key is test_row_0/C:col10/1731292320348/Put/seqid=0 2024-11-11T02:32:01,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742077_1253 (size=12001) 2024-11-11T02:32:01,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/5eaf09d21f9d412694b4bdda17d2163f 2024-11-11T02:32:01,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/dc426f62c8034bdba97bdafc08e4a05a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/dc426f62c8034bdba97bdafc08e4a05a 2024-11-11T02:32:01,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/dc426f62c8034bdba97bdafc08e4a05a, entries=200, sequenceid=118, filesize=14.0 K 2024-11-11T02:32:01,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/7275f8876a69447087beafef15a73318 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/7275f8876a69447087beafef15a73318 2024-11-11T02:32:01,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/7275f8876a69447087beafef15a73318, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T02:32:01,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/5eaf09d21f9d412694b4bdda17d2163f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/5eaf09d21f9d412694b4bdda17d2163f 2024-11-11T02:32:01,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/5eaf09d21f9d412694b4bdda17d2163f, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T02:32:01,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 3f251ac37bffb103f032beac897bfe5d in 869ms, sequenceid=118, compaction requested=true 2024-11-11T02:32:01,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:01,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:32:01,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:01,218 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:32:01,219 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:32:01,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:32:01,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:01,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:32:01,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:01,219 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:32:01,220 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/B is initiating minor compaction (all files) 2024-11-11T02:32:01,220 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:32:01,220 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/B in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:01,220 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/A is initiating minor compaction (all files) 2024-11-11T02:32:01,220 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/A in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:01,220 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/98655c671ebf4ce0811cf027f522bdce, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/17689d5f56fc413ea0fe1549fac6a05b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/489b4e7869474f5dbd1ed8b66371ecdb, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/7275f8876a69447087beafef15a73318] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=47.0 K 2024-11-11T02:32:01,220 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/4df0a5f8291449c3a87e825d017b374e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/93bce23e0e044e5491b91ef25d9d9158, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/48aacb08a91d4f1aafb560f21b905ce1, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/dc426f62c8034bdba97bdafc08e4a05a] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=49.3 K 2024-11-11T02:32:01,220 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 98655c671ebf4ce0811cf027f522bdce, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731292315441 2024-11-11T02:32:01,220 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4df0a5f8291449c3a87e825d017b374e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731292315441 2024-11-11T02:32:01,220 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 17689d5f56fc413ea0fe1549fac6a05b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731292316613 2024-11-11T02:32:01,220 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93bce23e0e044e5491b91ef25d9d9158, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731292316613 2024-11-11T02:32:01,220 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 489b4e7869474f5dbd1ed8b66371ecdb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731292318536 2024-11-11T02:32:01,221 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48aacb08a91d4f1aafb560f21b905ce1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731292318536 2024-11-11T02:32:01,222 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 7275f8876a69447087beafef15a73318, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292319718 2024-11-11T02:32:01,222 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc426f62c8034bdba97bdafc08e4a05a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292319714 2024-11-11T02:32:01,230 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#B#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:01,230 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/8ee882b6afa8448eb0cdea4eacf47a52 is 50, key is test_row_0/B:col10/1731292320348/Put/seqid=0 2024-11-11T02:32:01,231 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#A#compaction#207 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:01,232 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/6ccedf6bc61344f2b7b825bc34c4703e is 50, key is test_row_0/A:col10/1731292320348/Put/seqid=0 2024-11-11T02:32:01,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742078_1254 (size=12241) 2024-11-11T02:32:01,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742079_1255 (size=12241) 2024-11-11T02:32:01,245 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/6ccedf6bc61344f2b7b825bc34c4703e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/6ccedf6bc61344f2b7b825bc34c4703e 2024-11-11T02:32:01,251 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/A of 3f251ac37bffb103f032beac897bfe5d into 6ccedf6bc61344f2b7b825bc34c4703e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:01,251 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:01,251 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/A, priority=12, startTime=1731292321218; duration=0sec 2024-11-11T02:32:01,252 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:01,252 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:A 2024-11-11T02:32:01,252 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:32:01,253 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:32:01,253 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/C is initiating minor compaction (all files) 2024-11-11T02:32:01,253 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/C in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:01,253 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/47c719b86c3f4df3b6a01e8d5095ce23, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/c583220ac66c48e88ab73795096445e2, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8d2826ea829f47c18129a27fdf749bb4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/5eaf09d21f9d412694b4bdda17d2163f] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=47.0 K 2024-11-11T02:32:01,253 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47c719b86c3f4df3b6a01e8d5095ce23, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731292315441 2024-11-11T02:32:01,253 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting c583220ac66c48e88ab73795096445e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731292316613 2024-11-11T02:32:01,254 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d2826ea829f47c18129a27fdf749bb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731292318536 2024-11-11T02:32:01,254 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5eaf09d21f9d412694b4bdda17d2163f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292319718 2024-11-11T02:32:01,262 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#C#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:01,263 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/1f1134d136684a62add1167f70ede5ee is 50, key is test_row_0/C:col10/1731292320348/Put/seqid=0 2024-11-11T02:32:01,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742080_1256 (size=12241) 2024-11-11T02:32:01,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T02:32:01,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:01,325 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T02:32:01,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:32:01,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:01,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:32:01,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:01,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:32:01,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:01,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/a10396b53f534737960c11f404ec31cb is 50, key is test_row_0/A:col10/1731292320356/Put/seqid=0 2024-11-11T02:32:01,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742081_1257 (size=12001) 2024-11-11T02:32:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:01,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:01,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292321555, max timestamp=1731292321577Num uniques 859; 2024-11-11T02:32:01,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:32:01,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292321555, max timestamp=1731292321577Num uniques 859; 2024-11-11T02:32:01,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:32:01,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292321555, max timestamp=1731292321577Num uniques 859; 2024-11-11T02:32:01,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:32:01,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292381616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:01,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292381620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:01,638 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/8ee882b6afa8448eb0cdea4eacf47a52 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/8ee882b6afa8448eb0cdea4eacf47a52 2024-11-11T02:32:01,643 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/B of 3f251ac37bffb103f032beac897bfe5d into 8ee882b6afa8448eb0cdea4eacf47a52(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:01,643 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:01,643 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/B, priority=12, startTime=1731292321218; duration=0sec 2024-11-11T02:32:01,643 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:01,643 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:B 2024-11-11T02:32:01,683 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/1f1134d136684a62add1167f70ede5ee as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/1f1134d136684a62add1167f70ede5ee 2024-11-11T02:32:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T02:32:01,688 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/C of 3f251ac37bffb103f032beac897bfe5d into 1f1134d136684a62add1167f70ede5ee(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:01,688 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:01,688 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/C, priority=12, startTime=1731292321219; duration=0sec 2024-11-11T02:32:01,688 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:01,688 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:C 2024-11-11T02:32:01,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292381725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:01,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:01,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292381735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:01,740 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/a10396b53f534737960c11f404ec31cb 2024-11-11T02:32:01,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/8ed0ac7691d7490791462808641aabdf is 50, key is test_row_0/B:col10/1731292320356/Put/seqid=0 2024-11-11T02:32:01,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742082_1258 (size=12001) 2024-11-11T02:32:01,755 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/8ed0ac7691d7490791462808641aabdf 2024-11-11T02:32:01,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/84e69a41643d49b493989adf8c381e6a is 50, key is test_row_0/C:col10/1731292320356/Put/seqid=0 2024-11-11T02:32:01,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742083_1259 (size=12001) 2024-11-11T02:32:01,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292381936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:01,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292381945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:02,167 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/84e69a41643d49b493989adf8c381e6a 2024-11-11T02:32:02,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/a10396b53f534737960c11f404ec31cb as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/a10396b53f534737960c11f404ec31cb 2024-11-11T02:32:02,176 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/a10396b53f534737960c11f404ec31cb, entries=150, sequenceid=130, filesize=11.7 K 2024-11-11T02:32:02,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/8ed0ac7691d7490791462808641aabdf as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/8ed0ac7691d7490791462808641aabdf 2024-11-11T02:32:02,181 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/8ed0ac7691d7490791462808641aabdf, entries=150, sequenceid=130, filesize=11.7 K 2024-11-11T02:32:02,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/84e69a41643d49b493989adf8c381e6a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/84e69a41643d49b493989adf8c381e6a 2024-11-11T02:32:02,186 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/84e69a41643d49b493989adf8c381e6a, entries=150, sequenceid=130, filesize=11.7 K 2024-11-11T02:32:02,187 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3f251ac37bffb103f032beac897bfe5d in 861ms, sequenceid=130, compaction requested=false 2024-11-11T02:32:02,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:02,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:02,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-11T02:32:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-11T02:32:02,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-11T02:32:02,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6400 sec 2024-11-11T02:32:02,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.6440 sec 2024-11-11T02:32:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:02,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T02:32:02,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:32:02,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:02,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:32:02,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:02,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:32:02,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:02,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/35ac6d0d2c3448bdb4231d62059d7acf is 50, key is test_row_0/A:col10/1731292321615/Put/seqid=0 2024-11-11T02:32:02,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742084_1260 (size=12151) 2024-11-11T02:32:02,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:02,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292382275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292382275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:02,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:02,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:02,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292382385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:02,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:02,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292382385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:02,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:02,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292382595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:02,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:02,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292382595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:02,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/35ac6d0d2c3448bdb4231d62059d7acf 2024-11-11T02:32:02,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/67dee509dc624f66b9c7cb037588bfbc is 50, key is test_row_0/B:col10/1731292321615/Put/seqid=0 2024-11-11T02:32:02,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742085_1261 (size=12151) 2024-11-11T02:32:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T02:32:02,705 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:02,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-11T02:32:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T02:32:02,707 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:02,708 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:02,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T02:32:02,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T02:32:02,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:02,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:02,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:02,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:02,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:02,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292382906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:02,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292382906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:03,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T02:32:03,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:03,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:03,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:03,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:03,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T02:32:03,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/67dee509dc624f66b9c7cb037588bfbc 2024-11-11T02:32:03,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/c6236809d3134883b399784bf0c0b83e is 50, key is test_row_0/C:col10/1731292321615/Put/seqid=0 2024-11-11T02:32:03,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742086_1262 (size=12151) 2024-11-11T02:32:03,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T02:32:03,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:03,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:03,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:03,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:03,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:03,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T02:32:03,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:03,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:03,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:03,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:03,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T02:32:03,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292383415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:03,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:03,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:03,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:03,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292383415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:03,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:03,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:03,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:03,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T02:32:03,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:03,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:03,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:03,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:03,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:03,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/c6236809d3134883b399784bf0c0b83e 2024-11-11T02:32:03,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/35ac6d0d2c3448bdb4231d62059d7acf as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/35ac6d0d2c3448bdb4231d62059d7acf 2024-11-11T02:32:03,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/35ac6d0d2c3448bdb4231d62059d7acf, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T02:32:03,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/67dee509dc624f66b9c7cb037588bfbc as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/67dee509dc624f66b9c7cb037588bfbc 2024-11-11T02:32:03,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/67dee509dc624f66b9c7cb037588bfbc, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T02:32:03,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/c6236809d3134883b399784bf0c0b83e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/c6236809d3134883b399784bf0c0b83e 2024-11-11T02:32:03,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/c6236809d3134883b399784bf0c0b83e, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T02:32:03,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 3f251ac37bffb103f032beac897bfe5d in 1248ms, sequenceid=169, compaction requested=true 2024-11-11T02:32:03,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:03,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:32:03,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:03,498 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:03,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:32:03,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:03,498 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:03,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:32:03,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:03,499 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:03,499 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:03,499 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/B is initiating minor compaction (all files) 2024-11-11T02:32:03,499 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/A is initiating minor compaction (all files) 2024-11-11T02:32:03,500 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/B in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:03,500 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/A in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:03,500 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/8ee882b6afa8448eb0cdea4eacf47a52, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/8ed0ac7691d7490791462808641aabdf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/67dee509dc624f66b9c7cb037588bfbc] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=35.5 K 2024-11-11T02:32:03,500 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/6ccedf6bc61344f2b7b825bc34c4703e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/a10396b53f534737960c11f404ec31cb, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/35ac6d0d2c3448bdb4231d62059d7acf] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=35.5 K 2024-11-11T02:32:03,500 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ee882b6afa8448eb0cdea4eacf47a52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292319718 2024-11-11T02:32:03,500 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ccedf6bc61344f2b7b825bc34c4703e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292319718 2024-11-11T02:32:03,500 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting a10396b53f534737960c11f404ec31cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731292320356 2024-11-11T02:32:03,500 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ed0ac7691d7490791462808641aabdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731292320356 2024-11-11T02:32:03,501 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35ac6d0d2c3448bdb4231d62059d7acf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731292321606 2024-11-11T02:32:03,501 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 67dee509dc624f66b9c7cb037588bfbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731292321606 2024-11-11T02:32:03,507 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#A#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:03,508 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#B#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:03,508 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/ba456d3e2cdc40f7b3ff7f5b7b4be5fa is 50, key is test_row_0/B:col10/1731292321615/Put/seqid=0 2024-11-11T02:32:03,508 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/01235c647ca1423fb63e970439a0a489 is 50, key is test_row_0/A:col10/1731292321615/Put/seqid=0 2024-11-11T02:32:03,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742087_1263 (size=12493) 2024-11-11T02:32:03,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742088_1264 (size=12493) 2024-11-11T02:32:03,528 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/ba456d3e2cdc40f7b3ff7f5b7b4be5fa as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/ba456d3e2cdc40f7b3ff7f5b7b4be5fa 2024-11-11T02:32:03,532 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/B of 3f251ac37bffb103f032beac897bfe5d into ba456d3e2cdc40f7b3ff7f5b7b4be5fa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:03,532 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:03,532 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/B, priority=13, startTime=1731292323498; duration=0sec 2024-11-11T02:32:03,532 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:03,532 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:B 2024-11-11T02:32:03,532 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:03,533 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:03,533 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/C is initiating minor compaction (all files) 2024-11-11T02:32:03,533 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/C in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:03,533 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/1f1134d136684a62add1167f70ede5ee, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/84e69a41643d49b493989adf8c381e6a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/c6236809d3134883b399784bf0c0b83e] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=35.5 K 2024-11-11T02:32:03,534 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f1134d136684a62add1167f70ede5ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292319718 2024-11-11T02:32:03,534 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 84e69a41643d49b493989adf8c381e6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731292320356 2024-11-11T02:32:03,534 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting c6236809d3134883b399784bf0c0b83e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731292321606 2024-11-11T02:32:03,541 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:03,542 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/8885cc2487634b00973d0fe12fee18c8 is 50, key is test_row_0/C:col10/1731292321615/Put/seqid=0 2024-11-11T02:32:03,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742089_1265 (size=12493) 2024-11-11T02:32:03,551 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/8885cc2487634b00973d0fe12fee18c8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8885cc2487634b00973d0fe12fee18c8 2024-11-11T02:32:03,556 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/C of 3f251ac37bffb103f032beac897bfe5d into 8885cc2487634b00973d0fe12fee18c8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:03,557 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:03,557 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/C, priority=13, startTime=1731292323498; duration=0sec 2024-11-11T02:32:03,557 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:03,557 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:C 2024-11-11T02:32:03,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T02:32:03,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:03,624 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:32:03,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:32:03,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:03,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:32:03,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:03,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:32:03,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:03,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/93e4076ec63f4fad9fcf4dbf01f875a0 is 50, key is test_row_0/A:col10/1731292322273/Put/seqid=0 2024-11-11T02:32:03,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742090_1266 (size=12151) 2024-11-11T02:32:03,635 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/93e4076ec63f4fad9fcf4dbf01f875a0 2024-11-11T02:32:03,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/c2d24129f1834bc88f58e822c437f254 is 50, key is test_row_0/B:col10/1731292322273/Put/seqid=0 2024-11-11T02:32:03,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742091_1267 (size=12151) 2024-11-11T02:32:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T02:32:03,928 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/01235c647ca1423fb63e970439a0a489 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/01235c647ca1423fb63e970439a0a489 2024-11-11T02:32:03,932 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/A of 3f251ac37bffb103f032beac897bfe5d into 01235c647ca1423fb63e970439a0a489(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:03,932 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:03,932 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/A, priority=13, startTime=1731292323498; duration=0sec 2024-11-11T02:32:03,932 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:03,932 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:A 2024-11-11T02:32:04,048 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/c2d24129f1834bc88f58e822c437f254 2024-11-11T02:32:04,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/31d45b43804142b4be10ceb255cee927 is 50, key is test_row_0/C:col10/1731292322273/Put/seqid=0 2024-11-11T02:32:04,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742092_1268 (size=12151) 2024-11-11T02:32:04,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:04,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:04,459 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/31d45b43804142b4be10ceb255cee927 2024-11-11T02:32:04,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/93e4076ec63f4fad9fcf4dbf01f875a0 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/93e4076ec63f4fad9fcf4dbf01f875a0 2024-11-11T02:32:04,468 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/93e4076ec63f4fad9fcf4dbf01f875a0, entries=150, sequenceid=181, filesize=11.9 K 2024-11-11T02:32:04,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/c2d24129f1834bc88f58e822c437f254 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/c2d24129f1834bc88f58e822c437f254 2024-11-11T02:32:04,472 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/c2d24129f1834bc88f58e822c437f254, entries=150, sequenceid=181, filesize=11.9 K 2024-11-11T02:32:04,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/31d45b43804142b4be10ceb255cee927 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/31d45b43804142b4be10ceb255cee927 2024-11-11T02:32:04,477 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/31d45b43804142b4be10ceb255cee927, entries=150, sequenceid=181, filesize=11.9 K 2024-11-11T02:32:04,477 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=114.05 KB/116790 for 3f251ac37bffb103f032beac897bfe5d in 853ms, sequenceid=181, compaction requested=false 2024-11-11T02:32:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-11T02:32:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-11T02:32:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:04,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-11T02:32:04,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:32:04,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:04,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:32:04,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:04,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:32:04,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:04,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-11T02:32:04,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7710 sec 2024-11-11T02:32:04,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.7780 sec 2024-11-11T02:32:04,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/595b52a88057475e889b3af6efc90e9f is 50, key is test_row_0/A:col10/1731292324466/Put/seqid=0 2024-11-11T02:32:04,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742093_1269 (size=12151) 2024-11-11T02:32:04,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292384499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:04,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:04,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292384501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:04,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:04,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:04,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292384616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:04,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292384616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:04,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:04,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50550 deadline: 1731292384815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:04,819 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in 3f251ac37bffb103f032beac897bfe5d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:32:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:04,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:04,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1731292384826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:04,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292384826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,827 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in 3f251ac37bffb103f032beac897bfe5d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:32:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292384826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:04,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:04,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50490 deadline: 1731292384835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:04,839 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in 3f251ac37bffb103f032beac897bfe5d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:32:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:04,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T02:32:04,856 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:04,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:04,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-11T02:32:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T02:32:04,858 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:04,858 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:04,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:04,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/595b52a88057475e889b3af6efc90e9f 2024-11-11T02:32:04,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/bb30f5a899f049609678630c20ef1e77 is 50, key is test_row_0/B:col10/1731292324466/Put/seqid=0 2024-11-11T02:32:04,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742094_1270 (size=12151) 2024-11-11T02:32:04,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T02:32:05,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T02:32:05,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:05,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:05,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:05,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292385136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:05,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292385136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:05,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T02:32:05,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:05,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:05,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:05,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T02:32:05,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/bb30f5a899f049609678630c20ef1e77 2024-11-11T02:32:05,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/688a0172f9dd4f98be672af8fac5bb6d is 50, key is test_row_0/C:col10/1731292324466/Put/seqid=0 2024-11-11T02:32:05,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742095_1271 (size=12151) 2024-11-11T02:32:05,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T02:32:05,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:05,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:05,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:05,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T02:32:05,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:05,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:05,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:05,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T02:32:05,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T02:32:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:05,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:05,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292385645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:05,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:05,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:05,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:05,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292385645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:05,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:05,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:05,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/688a0172f9dd4f98be672af8fac5bb6d 2024-11-11T02:32:05,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/595b52a88057475e889b3af6efc90e9f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/595b52a88057475e889b3af6efc90e9f 2024-11-11T02:32:05,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/595b52a88057475e889b3af6efc90e9f, entries=150, sequenceid=204, filesize=11.9 K 2024-11-11T02:32:05,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/bb30f5a899f049609678630c20ef1e77 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/bb30f5a899f049609678630c20ef1e77 2024-11-11T02:32:05,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/bb30f5a899f049609678630c20ef1e77, entries=150, sequenceid=204, filesize=11.9 K 2024-11-11T02:32:05,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/688a0172f9dd4f98be672af8fac5bb6d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/688a0172f9dd4f98be672af8fac5bb6d 2024-11-11T02:32:05,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/688a0172f9dd4f98be672af8fac5bb6d, entries=150, sequenceid=204, filesize=11.9 K 2024-11-11T02:32:05,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 3f251ac37bffb103f032beac897bfe5d in 1251ms, sequenceid=204, compaction requested=true 2024-11-11T02:32:05,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:05,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:32:05,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:05,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:32:05,732 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:05,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:05,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:32:05,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:05,732 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:05,732 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:05,732 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:05,732 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/B is initiating minor compaction (all files) 2024-11-11T02:32:05,732 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/A is initiating minor compaction (all files) 2024-11-11T02:32:05,732 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/A in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:05,732 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/B in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:05,733 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/01235c647ca1423fb63e970439a0a489, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/93e4076ec63f4fad9fcf4dbf01f875a0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/595b52a88057475e889b3af6efc90e9f] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=35.9 K 2024-11-11T02:32:05,733 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/ba456d3e2cdc40f7b3ff7f5b7b4be5fa, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/c2d24129f1834bc88f58e822c437f254, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/bb30f5a899f049609678630c20ef1e77] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=35.9 K 2024-11-11T02:32:05,733 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting ba456d3e2cdc40f7b3ff7f5b7b4be5fa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731292321606 2024-11-11T02:32:05,733 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting c2d24129f1834bc88f58e822c437f254, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731292322265 2024-11-11T02:32:05,733 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01235c647ca1423fb63e970439a0a489, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731292321606 2024-11-11T02:32:05,734 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting bb30f5a899f049609678630c20ef1e77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292324466 2024-11-11T02:32:05,734 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93e4076ec63f4fad9fcf4dbf01f875a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731292322265 2024-11-11T02:32:05,735 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 595b52a88057475e889b3af6efc90e9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292324466 2024-11-11T02:32:05,741 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#B#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:05,741 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/5446155d5c6a4317937224d3173e6d57 is 50, key is test_row_0/B:col10/1731292324466/Put/seqid=0 2024-11-11T02:32:05,741 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#A#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:05,742 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/f8f945956d0949cbb15a6c84ef32219c is 50, key is test_row_0/A:col10/1731292324466/Put/seqid=0 2024-11-11T02:32:05,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742096_1272 (size=12595) 2024-11-11T02:32:05,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742097_1273 (size=12595) 2024-11-11T02:32:05,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T02:32:05,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:05,774 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:32:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:32:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:32:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:32:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:05,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/1b5ce95b013a4e21afacac177257233c is 50, key is test_row_0/A:col10/1731292324486/Put/seqid=0 2024-11-11T02:32:05,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742098_1274 (size=12151) 2024-11-11T02:32:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T02:32:06,150 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/5446155d5c6a4317937224d3173e6d57 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/5446155d5c6a4317937224d3173e6d57 2024-11-11T02:32:06,154 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/B of 3f251ac37bffb103f032beac897bfe5d into 5446155d5c6a4317937224d3173e6d57(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:06,154 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:06,154 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/B, priority=13, startTime=1731292325732; duration=0sec 2024-11-11T02:32:06,155 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:06,155 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:B 2024-11-11T02:32:06,155 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:06,155 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:06,155 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/C is initiating minor compaction (all files) 2024-11-11T02:32:06,156 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/C in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:06,156 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8885cc2487634b00973d0fe12fee18c8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/31d45b43804142b4be10ceb255cee927, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/688a0172f9dd4f98be672af8fac5bb6d] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=35.9 K 2024-11-11T02:32:06,156 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 8885cc2487634b00973d0fe12fee18c8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731292321606 2024-11-11T02:32:06,156 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 31d45b43804142b4be10ceb255cee927, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731292322265 2024-11-11T02:32:06,157 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 688a0172f9dd4f98be672af8fac5bb6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292324466 2024-11-11T02:32:06,158 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/f8f945956d0949cbb15a6c84ef32219c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/f8f945956d0949cbb15a6c84ef32219c 2024-11-11T02:32:06,162 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/A of 3f251ac37bffb103f032beac897bfe5d into f8f945956d0949cbb15a6c84ef32219c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:06,162 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:06,162 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/A, priority=13, startTime=1731292325731; duration=0sec 2024-11-11T02:32:06,162 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:06,162 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:A 2024-11-11T02:32:06,162 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#C#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:06,163 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/108f1e8b386e42199b35ba0b3bcecc5e is 50, key is test_row_0/C:col10/1731292324466/Put/seqid=0 2024-11-11T02:32:06,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742099_1275 (size=12595) 2024-11-11T02:32:06,183 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/1b5ce95b013a4e21afacac177257233c 2024-11-11T02:32:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/c872f95896014bd3986147d9c81840c0 is 50, key is test_row_0/B:col10/1731292324486/Put/seqid=0 2024-11-11T02:32:06,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742100_1276 (size=12151) 2024-11-11T02:32:06,571 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/108f1e8b386e42199b35ba0b3bcecc5e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/108f1e8b386e42199b35ba0b3bcecc5e 2024-11-11T02:32:06,576 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/C of 3f251ac37bffb103f032beac897bfe5d into 108f1e8b386e42199b35ba0b3bcecc5e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:06,576 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:06,577 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/C, priority=13, startTime=1731292325732; duration=0sec 2024-11-11T02:32:06,577 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:06,577 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:C 2024-11-11T02:32:06,597 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/c872f95896014bd3986147d9c81840c0 2024-11-11T02:32:06,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/a59b3f300be144f2a4e92bacd1f3db18 is 50, key is test_row_0/C:col10/1731292324486/Put/seqid=0 2024-11-11T02:32:06,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742101_1277 (size=12151) 2024-11-11T02:32:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:06,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:06,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292326701, max timestamp=1731292326717Num uniques 859; 2024-11-11T02:32:06,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:32:06,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292326701, max timestamp=1731292326717Num uniques 859; 2024-11-11T02:32:06,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292326701, max timestamp=1731292326717Num uniques 859; 2024-11-11T02:32:06,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:32:06,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:32:06,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292386756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:06,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292386760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:06,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292386866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:06,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292386866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:07,012 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/a59b3f300be144f2a4e92bacd1f3db18 2024-11-11T02:32:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T02:32:07,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/1b5ce95b013a4e21afacac177257233c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/1b5ce95b013a4e21afacac177257233c 2024-11-11T02:32:07,020 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/1b5ce95b013a4e21afacac177257233c, entries=150, sequenceid=214, filesize=11.9 K 2024-11-11T02:32:07,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/c872f95896014bd3986147d9c81840c0 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/c872f95896014bd3986147d9c81840c0 2024-11-11T02:32:07,024 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/c872f95896014bd3986147d9c81840c0, entries=150, sequenceid=214, filesize=11.9 K 2024-11-11T02:32:07,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/a59b3f300be144f2a4e92bacd1f3db18 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a59b3f300be144f2a4e92bacd1f3db18 2024-11-11T02:32:07,028 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a59b3f300be144f2a4e92bacd1f3db18, entries=150, sequenceid=214, filesize=11.9 K 2024-11-11T02:32:07,030 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3f251ac37bffb103f032beac897bfe5d in 1256ms, sequenceid=214, compaction requested=false 2024-11-11T02:32:07,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:07,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:07,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-11T02:32:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-11T02:32:07,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-11T02:32:07,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1730 sec 2024-11-11T02:32:07,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.1770 sec 2024-11-11T02:32:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:07,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-11T02:32:07,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:32:07,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:32:07,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:32:07,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:32:07,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:32:07,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:32:07,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/1796f1abb0934fb4b5936dae65f47efa is 50, key is test_row_0/A:col10/1731292326758/Put/seqid=0 2024-11-11T02:32:07,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742102_1278 (size=14541) 2024-11-11T02:32:07,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292387093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:07,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292387097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:07,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:07,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292387206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:07,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292387206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:07,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292387415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:07,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292387415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:07,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/1796f1abb0934fb4b5936dae65f47efa 2024-11-11T02:32:07,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/483c0576058448ad8d22cb8fd6d133af is 50, key is test_row_0/B:col10/1731292326758/Put/seqid=0 2024-11-11T02:32:07,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742103_1279 (size=12151) 2024-11-11T02:32:07,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292387726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:07,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292387726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:07,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/483c0576058448ad8d22cb8fd6d133af 2024-11-11T02:32:07,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/a56bdc6ef2e441059dce6ebc12c02a7d is 50, key is test_row_0/C:col10/1731292326758/Put/seqid=0 2024-11-11T02:32:07,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742104_1280 (size=12151) 2024-11-11T02:32:08,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292388235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:08,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292388236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:08,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/a56bdc6ef2e441059dce6ebc12c02a7d 2024-11-11T02:32:08,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/1796f1abb0934fb4b5936dae65f47efa as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/1796f1abb0934fb4b5936dae65f47efa 2024-11-11T02:32:08,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/1796f1abb0934fb4b5936dae65f47efa, entries=200, sequenceid=255, filesize=14.2 K 2024-11-11T02:32:08,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/483c0576058448ad8d22cb8fd6d133af as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/483c0576058448ad8d22cb8fd6d133af 2024-11-11T02:32:08,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/483c0576058448ad8d22cb8fd6d133af, entries=150, sequenceid=255, filesize=11.9 K 2024-11-11T02:32:08,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/a56bdc6ef2e441059dce6ebc12c02a7d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a56bdc6ef2e441059dce6ebc12c02a7d 2024-11-11T02:32:08,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a56bdc6ef2e441059dce6ebc12c02a7d, entries=150, sequenceid=255, filesize=11.9 K 2024-11-11T02:32:08,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 3f251ac37bffb103f032beac897bfe5d in 1247ms, sequenceid=255, compaction requested=true 2024-11-11T02:32:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:32:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:08,326 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:32:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:08,326 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:32:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:08,327 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:08,327 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:08,327 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/B is initiating minor compaction (all files) 2024-11-11T02:32:08,327 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/A is initiating minor compaction (all files) 2024-11-11T02:32:08,327 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/B in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:08,327 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/A in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:08,327 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/5446155d5c6a4317937224d3173e6d57, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/c872f95896014bd3986147d9c81840c0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/483c0576058448ad8d22cb8fd6d133af] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=36.0 K 2024-11-11T02:32:08,327 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/f8f945956d0949cbb15a6c84ef32219c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/1b5ce95b013a4e21afacac177257233c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/1796f1abb0934fb4b5936dae65f47efa] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=38.4 K 2024-11-11T02:32:08,327 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 5446155d5c6a4317937224d3173e6d57, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292324466 2024-11-11T02:32:08,327 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8f945956d0949cbb15a6c84ef32219c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292324466 2024-11-11T02:32:08,328 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting c872f95896014bd3986147d9c81840c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731292324486 2024-11-11T02:32:08,328 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b5ce95b013a4e21afacac177257233c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731292324486 2024-11-11T02:32:08,328 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 483c0576058448ad8d22cb8fd6d133af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731292326753 2024-11-11T02:32:08,328 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1796f1abb0934fb4b5936dae65f47efa, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731292326753 2024-11-11T02:32:08,343 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#B#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:08,344 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#A#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:08,344 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/6963816c671b4f438f1f384d81e36a5b is 50, key is test_row_0/B:col10/1731292326758/Put/seqid=0 2024-11-11T02:32:08,344 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/b60c963c16e04c3096c21ec00598eb0e is 50, key is test_row_0/A:col10/1731292326758/Put/seqid=0 2024-11-11T02:32:08,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742105_1281 (size=12697) 2024-11-11T02:32:08,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742106_1282 (size=12697) 2024-11-11T02:32:08,361 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/b60c963c16e04c3096c21ec00598eb0e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b60c963c16e04c3096c21ec00598eb0e 2024-11-11T02:32:08,365 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/A of 3f251ac37bffb103f032beac897bfe5d into b60c963c16e04c3096c21ec00598eb0e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:08,365 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:08,365 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/A, priority=13, startTime=1731292328326; duration=0sec 2024-11-11T02:32:08,366 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:08,366 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:A 2024-11-11T02:32:08,366 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:08,366 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:08,366 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/C is initiating minor compaction (all files) 2024-11-11T02:32:08,366 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/C in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:08,367 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/108f1e8b386e42199b35ba0b3bcecc5e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a59b3f300be144f2a4e92bacd1f3db18, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a56bdc6ef2e441059dce6ebc12c02a7d] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=36.0 K 2024-11-11T02:32:08,367 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 108f1e8b386e42199b35ba0b3bcecc5e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292324466 2024-11-11T02:32:08,368 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting a59b3f300be144f2a4e92bacd1f3db18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731292324486 2024-11-11T02:32:08,368 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56bdc6ef2e441059dce6ebc12c02a7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731292326753 2024-11-11T02:32:08,376 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#C#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:08,378 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/944be338acb149b29f038a5880816955 is 50, key is test_row_0/C:col10/1731292326758/Put/seqid=0 2024-11-11T02:32:08,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742107_1283 (size=12697) 2024-11-11T02:32:08,389 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/944be338acb149b29f038a5880816955 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/944be338acb149b29f038a5880816955 2024-11-11T02:32:08,395 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/C of 3f251ac37bffb103f032beac897bfe5d into 944be338acb149b29f038a5880816955(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:08,395 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:08,395 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/C, priority=13, startTime=1731292328326; duration=0sec 2024-11-11T02:32:08,395 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:08,396 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:C 2024-11-11T02:32:08,761 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/6963816c671b4f438f1f384d81e36a5b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/6963816c671b4f438f1f384d81e36a5b 2024-11-11T02:32:08,766 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/B of 3f251ac37bffb103f032beac897bfe5d into 6963816c671b4f438f1f384d81e36a5b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:08,766 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:08,766 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/B, priority=13, startTime=1731292328326; duration=0sec 2024-11-11T02:32:08,767 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:08,767 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:B 2024-11-11T02:32:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T02:32:09,035 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:09,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-11T02:32:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T02:32:09,037 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:09,038 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:09,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T02:32:09,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T02:32:09,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:09,190 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T02:32:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:32:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:32:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:32:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:09,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/77a3e35e28cc469d8c42fde29003b57d is 50, key is test_row_0/A:col10/1731292327080/Put/seqid=0 2024-11-11T02:32:09,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742108_1284 (size=12301) 2024-11-11T02:32:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:09,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:09,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292329273, max timestamp=1731292329297Num uniques 859; 2024-11-11T02:32:09,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:09,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292329273, max timestamp=1731292329297Num uniques 859; 2024-11-11T02:32:09,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:09,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292329273, max timestamp=1731292329297Num uniques 859; 2024-11-11T02:32:09,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:09,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292389346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:09,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292389348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T02:32:09,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292389455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:09,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292389456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:09,605 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/77a3e35e28cc469d8c42fde29003b57d 2024-11-11T02:32:09,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/45ffd8c026b04121ba5ceaeb27c3a01e is 50, key is test_row_0/B:col10/1731292327080/Put/seqid=0 2024-11-11T02:32:09,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742109_1285 (size=12301) 2024-11-11T02:32:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T02:32:09,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292389665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:09,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292389666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:09,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292389976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292389976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:10,017 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/45ffd8c026b04121ba5ceaeb27c3a01e 2024-11-11T02:32:10,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/a714a89d0ef6411385768c589d4ea107 is 50, key is test_row_0/C:col10/1731292327080/Put/seqid=0 2024-11-11T02:32:10,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742110_1286 (size=12301) 2024-11-11T02:32:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T02:32:10,428 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/a714a89d0ef6411385768c589d4ea107 2024-11-11T02:32:10,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/77a3e35e28cc469d8c42fde29003b57d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/77a3e35e28cc469d8c42fde29003b57d 2024-11-11T02:32:10,436 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/77a3e35e28cc469d8c42fde29003b57d, entries=150, sequenceid=267, filesize=12.0 K 2024-11-11T02:32:10,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/45ffd8c026b04121ba5ceaeb27c3a01e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/45ffd8c026b04121ba5ceaeb27c3a01e 2024-11-11T02:32:10,441 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/45ffd8c026b04121ba5ceaeb27c3a01e, entries=150, sequenceid=267, filesize=12.0 K 2024-11-11T02:32:10,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/a714a89d0ef6411385768c589d4ea107 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a714a89d0ef6411385768c589d4ea107 2024-11-11T02:32:10,445 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a714a89d0ef6411385768c589d4ea107, entries=150, sequenceid=267, filesize=12.0 K 2024-11-11T02:32:10,446 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 3f251ac37bffb103f032beac897bfe5d in 1256ms, sequenceid=267, compaction requested=false 2024-11-11T02:32:10,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:10,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:10,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-11T02:32:10,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-11T02:32:10,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-11T02:32:10,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-11T02:32:10,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4130 sec 2024-11-11T02:32:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:10,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-11T02:32:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:32:10,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:32:10,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:32:10,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:32:10,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:32:10,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:32:10,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/b1bbae3edcd34f67890de8c442dc918b is 50, key is test_row_0/A:col10/1731292329338/Put/seqid=0 2024-11-11T02:32:10,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742111_1287 (size=14741) 2024-11-11T02:32:10,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292390500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:10,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292390505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:10,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:10,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:10,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:10,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292390616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:10,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292390616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:10,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292390825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:10,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292390825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:10,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/b1bbae3edcd34f67890de8c442dc918b 2024-11-11T02:32:10,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/457cf1890ab8491faf879915cc4e66ca is 50, key is test_row_0/B:col10/1731292329338/Put/seqid=0 2024-11-11T02:32:10,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742112_1288 (size=12301) 2024-11-11T02:32:11,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292391136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:11,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292391136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T02:32:11,185 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:11,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-11T02:32:11,187 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T02:32:11,188 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:11,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T02:32:11,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/457cf1890ab8491faf879915cc4e66ca 2024-11-11T02:32:11,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/19de398057a1480dbbc552d862ac5722 is 50, key is test_row_0/C:col10/1731292329338/Put/seqid=0 2024-11-11T02:32:11,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742113_1289 (size=12301) 2024-11-11T02:32:11,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T02:32:11,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:11,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:11,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:11,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:11,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:11,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T02:32:11,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:11,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:11,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:11,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:11,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T02:32:11,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T02:32:11,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:11,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:11,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:11,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:11,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:11,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292391646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:11,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292391646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:11,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:11,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:11,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:11,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:11,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:11,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:11,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/19de398057a1480dbbc552d862ac5722 2024-11-11T02:32:11,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/b1bbae3edcd34f67890de8c442dc918b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b1bbae3edcd34f67890de8c442dc918b 2024-11-11T02:32:11,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b1bbae3edcd34f67890de8c442dc918b, entries=200, sequenceid=305, filesize=14.4 K 2024-11-11T02:32:11,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/457cf1890ab8491faf879915cc4e66ca as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/457cf1890ab8491faf879915cc4e66ca 2024-11-11T02:32:11,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/457cf1890ab8491faf879915cc4e66ca, entries=150, sequenceid=305, filesize=12.0 K 2024-11-11T02:32:11,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/19de398057a1480dbbc552d862ac5722 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/19de398057a1480dbbc552d862ac5722 2024-11-11T02:32:11,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/19de398057a1480dbbc552d862ac5722, entries=150, sequenceid=305, filesize=12.0 K 2024-11-11T02:32:11,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 3f251ac37bffb103f032beac897bfe5d in 1253ms, sequenceid=305, compaction requested=true 2024-11-11T02:32:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:32:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:11,743 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:32:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:11,743 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:32:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:11,744 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:11,744 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:11,744 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/A is initiating minor compaction (all files) 2024-11-11T02:32:11,744 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/B is initiating minor compaction (all files) 2024-11-11T02:32:11,744 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/B in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:11,744 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/6963816c671b4f438f1f384d81e36a5b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/45ffd8c026b04121ba5ceaeb27c3a01e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/457cf1890ab8491faf879915cc4e66ca] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=36.4 K 2024-11-11T02:32:11,744 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/A in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:11,744 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b60c963c16e04c3096c21ec00598eb0e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/77a3e35e28cc469d8c42fde29003b57d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b1bbae3edcd34f67890de8c442dc918b] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=38.8 K 2024-11-11T02:32:11,744 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 6963816c671b4f438f1f384d81e36a5b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731292326753 2024-11-11T02:32:11,745 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting b60c963c16e04c3096c21ec00598eb0e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731292326753 2024-11-11T02:32:11,745 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 45ffd8c026b04121ba5ceaeb27c3a01e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731292327080 2024-11-11T02:32:11,745 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77a3e35e28cc469d8c42fde29003b57d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731292327080 2024-11-11T02:32:11,745 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 457cf1890ab8491faf879915cc4e66ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731292329338 2024-11-11T02:32:11,749 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1bbae3edcd34f67890de8c442dc918b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731292329338 2024-11-11T02:32:11,756 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#A#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:11,756 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:11,756 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/7995f781aa2349319d9e758b2ca20a72 is 50, key is test_row_0/A:col10/1731292329338/Put/seqid=0 2024-11-11T02:32:11,756 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/b46c32b881544b3195a59d225bc383e0 is 50, key is test_row_0/B:col10/1731292329338/Put/seqid=0 2024-11-11T02:32:11,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742115_1291 (size=12949) 2024-11-11T02:32:11,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742114_1290 (size=12949) 2024-11-11T02:32:11,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T02:32:11,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:11,798 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T02:32:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:32:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:32:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:32:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:11,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/eb78233ae24646eaabab8dd65dfc92fc is 50, key is test_row_0/A:col10/1731292330503/Put/seqid=0 2024-11-11T02:32:11,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742116_1292 (size=12301) 2024-11-11T02:32:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T02:32:12,173 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/7995f781aa2349319d9e758b2ca20a72 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/7995f781aa2349319d9e758b2ca20a72 2024-11-11T02:32:12,177 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/A of 3f251ac37bffb103f032beac897bfe5d into 7995f781aa2349319d9e758b2ca20a72(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:12,177 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:12,178 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/A, priority=13, startTime=1731292331743; duration=0sec 2024-11-11T02:32:12,178 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:12,178 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:A 2024-11-11T02:32:12,178 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:12,179 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:12,179 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/C is initiating minor compaction (all files) 2024-11-11T02:32:12,179 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/C in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:12,179 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/944be338acb149b29f038a5880816955, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a714a89d0ef6411385768c589d4ea107, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/19de398057a1480dbbc552d862ac5722] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=36.4 K 2024-11-11T02:32:12,179 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 944be338acb149b29f038a5880816955, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731292326753 2024-11-11T02:32:12,179 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting a714a89d0ef6411385768c589d4ea107, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731292327080 2024-11-11T02:32:12,180 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19de398057a1480dbbc552d862ac5722, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731292329338 2024-11-11T02:32:12,186 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#C#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:12,186 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/1455cffc6cec426ea1fb11a47caeed78 is 50, key is test_row_0/C:col10/1731292329338/Put/seqid=0 2024-11-11T02:32:12,187 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/b46c32b881544b3195a59d225bc383e0 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/b46c32b881544b3195a59d225bc383e0 2024-11-11T02:32:12,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742117_1293 (size=12949) 2024-11-11T02:32:12,196 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/B of 3f251ac37bffb103f032beac897bfe5d into b46c32b881544b3195a59d225bc383e0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:12,196 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:12,196 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/B, priority=13, startTime=1731292331743; duration=0sec 2024-11-11T02:32:12,196 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:12,196 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:B 2024-11-11T02:32:12,206 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/eb78233ae24646eaabab8dd65dfc92fc 2024-11-11T02:32:12,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/57d49f869e264459b08c604408fd1a22 is 50, key is test_row_0/B:col10/1731292330503/Put/seqid=0 2024-11-11T02:32:12,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742118_1294 (size=12301) 2024-11-11T02:32:12,216 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/57d49f869e264459b08c604408fd1a22 2024-11-11T02:32:12,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/6585b48cf0a34817a89e5fde82d77913 is 50, key is test_row_0/C:col10/1731292330503/Put/seqid=0 2024-11-11T02:32:12,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742119_1295 (size=12301) 2024-11-11T02:32:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T02:32:12,594 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/1455cffc6cec426ea1fb11a47caeed78 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/1455cffc6cec426ea1fb11a47caeed78 2024-11-11T02:32:12,598 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/C of 3f251ac37bffb103f032beac897bfe5d into 1455cffc6cec426ea1fb11a47caeed78(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:12,598 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:12,598 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/C, priority=13, startTime=1731292331743; duration=0sec 2024-11-11T02:32:12,598 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:12,598 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:C 2024-11-11T02:32:12,627 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/6585b48cf0a34817a89e5fde82d77913 2024-11-11T02:32:12,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/eb78233ae24646eaabab8dd65dfc92fc as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/eb78233ae24646eaabab8dd65dfc92fc 2024-11-11T02:32:12,634 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/eb78233ae24646eaabab8dd65dfc92fc, entries=150, sequenceid=313, filesize=12.0 K 2024-11-11T02:32:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/57d49f869e264459b08c604408fd1a22 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/57d49f869e264459b08c604408fd1a22 2024-11-11T02:32:12,638 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/57d49f869e264459b08c604408fd1a22, entries=150, sequenceid=313, filesize=12.0 K 2024-11-11T02:32:12,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/6585b48cf0a34817a89e5fde82d77913 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/6585b48cf0a34817a89e5fde82d77913 2024-11-11T02:32:12,642 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/6585b48cf0a34817a89e5fde82d77913, entries=150, sequenceid=313, filesize=12.0 K 2024-11-11T02:32:12,643 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3f251ac37bffb103f032beac897bfe5d in 845ms, sequenceid=313, compaction requested=false 2024-11-11T02:32:12,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:12,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:12,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-11T02:32:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-11T02:32:12,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-11T02:32:12,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4570 sec 2024-11-11T02:32:12,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.4610 sec 2024-11-11T02:32:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:12,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:32:12,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:32:12,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:12,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:32:12,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:12,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:32:12,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:12,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/b00022da1fb54f12a0ab10b2e3b029d7 is 50, key is test_row_0/A:col10/1731292332662/Put/seqid=0 2024-11-11T02:32:12,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742120_1296 (size=12301) 2024-11-11T02:32:12,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292332692, max timestamp=1731292332702Num uniques 859; 2024-11-11T02:32:12,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292332692, max timestamp=1731292332702Num uniques 859; 2024-11-11T02:32:12,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292332692, max timestamp=1731292332702Num uniques 859; 2024-11-11T02:32:12,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:32:12,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:32:12,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:32:12,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292392747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:12,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292392749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:12,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292392855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:12,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292392855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:13,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:13,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292393066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:13,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:13,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292393066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:13,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/b00022da1fb54f12a0ab10b2e3b029d7 2024-11-11T02:32:13,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/6258d816df2b4895a3702ca9dba194a5 is 50, key is test_row_0/B:col10/1731292332662/Put/seqid=0 2024-11-11T02:32:13,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742121_1297 (size=12301) 2024-11-11T02:32:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T02:32:13,335 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:13,336 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:13,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-11T02:32:13,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T02:32:13,337 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:13,338 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:13,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:13,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292393375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:13,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292393375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T02:32:13,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/6258d816df2b4895a3702ca9dba194a5 2024-11-11T02:32:13,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T02:32:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:13,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:13,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/0263f6da882643309de7b421d2be8b9a is 50, key is test_row_0/C:col10/1731292332662/Put/seqid=0 2024-11-11T02:32:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:13,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742122_1298 (size=12301) 2024-11-11T02:32:13,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T02:32:13,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:13,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:13,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:13,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:13,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T02:32:13,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T02:32:13,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:13,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:13,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:13,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:13,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:13,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:13,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731292393885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:13,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50468 deadline: 1731292393885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:13,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:13,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:13,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:13,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:13,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:13,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:13,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/0263f6da882643309de7b421d2be8b9a 2024-11-11T02:32:13,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/b00022da1fb54f12a0ab10b2e3b029d7 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b00022da1fb54f12a0ab10b2e3b029d7 2024-11-11T02:32:13,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b00022da1fb54f12a0ab10b2e3b029d7, entries=150, sequenceid=326, filesize=12.0 K 2024-11-11T02:32:13,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/6258d816df2b4895a3702ca9dba194a5 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/6258d816df2b4895a3702ca9dba194a5 2024-11-11T02:32:13,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/6258d816df2b4895a3702ca9dba194a5, entries=150, sequenceid=326, filesize=12.0 K 2024-11-11T02:32:13,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/0263f6da882643309de7b421d2be8b9a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/0263f6da882643309de7b421d2be8b9a 2024-11-11T02:32:13,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/0263f6da882643309de7b421d2be8b9a, entries=150, sequenceid=326, filesize=12.0 K 2024-11-11T02:32:13,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3f251ac37bffb103f032beac897bfe5d in 1246ms, sequenceid=326, compaction requested=true 2024-11-11T02:32:13,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:13,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:32:13,913 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:13,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:13,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:32:13,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:13,913 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:13,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f251ac37bffb103f032beac897bfe5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:32:13,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:13,914 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:13,914 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:13,914 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/A is initiating minor compaction (all files) 2024-11-11T02:32:13,914 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/B is initiating minor compaction (all files) 2024-11-11T02:32:13,914 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/A in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:13,914 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/B in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:13,914 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/7995f781aa2349319d9e758b2ca20a72, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/eb78233ae24646eaabab8dd65dfc92fc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b00022da1fb54f12a0ab10b2e3b029d7] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=36.7 K 2024-11-11T02:32:13,914 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/b46c32b881544b3195a59d225bc383e0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/57d49f869e264459b08c604408fd1a22, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/6258d816df2b4895a3702ca9dba194a5] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=36.7 K 2024-11-11T02:32:13,914 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7995f781aa2349319d9e758b2ca20a72, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731292329338 2024-11-11T02:32:13,914 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting b46c32b881544b3195a59d225bc383e0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731292329338 2024-11-11T02:32:13,915 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb78233ae24646eaabab8dd65dfc92fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731292330490 2024-11-11T02:32:13,915 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 57d49f869e264459b08c604408fd1a22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731292330490 2024-11-11T02:32:13,915 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting b00022da1fb54f12a0ab10b2e3b029d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731292332662 2024-11-11T02:32:13,915 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 6258d816df2b4895a3702ca9dba194a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731292332662 2024-11-11T02:32:13,921 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:13,921 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/7cdf4c7e56c346908498b9db342d07da is 50, key is test_row_0/A:col10/1731292332662/Put/seqid=0 2024-11-11T02:32:13,921 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:13,922 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/f51ccf69211d4583af35533e37ca84b6 is 50, key is test_row_0/B:col10/1731292332662/Put/seqid=0 2024-11-11T02:32:13,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742124_1300 (size=13051) 2024-11-11T02:32:13,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742123_1299 (size=13051) 2024-11-11T02:32:13,929 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/7cdf4c7e56c346908498b9db342d07da as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/7cdf4c7e56c346908498b9db342d07da 2024-11-11T02:32:13,933 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/A of 3f251ac37bffb103f032beac897bfe5d into 7cdf4c7e56c346908498b9db342d07da(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:13,933 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:13,933 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/A, priority=13, startTime=1731292333913; duration=0sec 2024-11-11T02:32:13,933 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:13,933 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:A 2024-11-11T02:32:13,933 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:13,936 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:13,936 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 3f251ac37bffb103f032beac897bfe5d/C is initiating minor compaction (all files) 2024-11-11T02:32:13,936 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f251ac37bffb103f032beac897bfe5d/C in TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:13,937 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/1455cffc6cec426ea1fb11a47caeed78, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/6585b48cf0a34817a89e5fde82d77913, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/0263f6da882643309de7b421d2be8b9a] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp, totalSize=36.7 K 2024-11-11T02:32:13,937 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1455cffc6cec426ea1fb11a47caeed78, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731292329338 2024-11-11T02:32:13,937 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6585b48cf0a34817a89e5fde82d77913, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731292330490 2024-11-11T02:32:13,937 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0263f6da882643309de7b421d2be8b9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731292332662 2024-11-11T02:32:13,943 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f251ac37bffb103f032beac897bfe5d#C#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:13,943 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/bb9da215dbf64dceac7f6dacd39b2ea7 is 50, key is test_row_0/C:col10/1731292332662/Put/seqid=0 2024-11-11T02:32:13,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742125_1301 (size=13051) 2024-11-11T02:32:13,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T02:32:13,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:13,949 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T02:32:13,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:32:13,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:13,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:32:13,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:13,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:32:13,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:13,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/f1eb1a0e5f7340c78a5029c5ed9fd6e7 is 50, key is test_row_0/A:col10/1731292332734/Put/seqid=0 2024-11-11T02:32:13,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742126_1302 (size=12301) 2024-11-11T02:32:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T02:32:14,237 INFO [Thread-1096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1096. 2024-11-11T02:32:14,237 DEBUG [Thread-1096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:14,237 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,238 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:14,239 INFO [Thread-1102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1102. 2024-11-11T02:32:14,239 INFO [Thread-1100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1100. 2024-11-11T02:32:14,239 DEBUG [Thread-1102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:14,239 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,239 DEBUG [Thread-1100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:14,239 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,239 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,239 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:14,239 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,239 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:14,241 INFO [Thread-1098 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1098. 2024-11-11T02:32:14,241 DEBUG [Thread-1098 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:14,241 DEBUG [Thread-1098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,241 DEBUG [Thread-1098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,241 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:14,243 INFO [Thread-1104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1104. 2024-11-11T02:32:14,243 DEBUG [Thread-1104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:14,243 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,243 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,243 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:14,334 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/f51ccf69211d4583af35533e37ca84b6 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/f51ccf69211d4583af35533e37ca84b6 2024-11-11T02:32:14,342 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/B of 3f251ac37bffb103f032beac897bfe5d into f51ccf69211d4583af35533e37ca84b6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:14,342 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:14,342 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/B, priority=13, startTime=1731292333913; duration=0sec 2024-11-11T02:32:14,342 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:14,342 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:B 2024-11-11T02:32:14,353 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/bb9da215dbf64dceac7f6dacd39b2ea7 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/bb9da215dbf64dceac7f6dacd39b2ea7 2024-11-11T02:32:14,358 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f251ac37bffb103f032beac897bfe5d/C of 3f251ac37bffb103f032beac897bfe5d into bb9da215dbf64dceac7f6dacd39b2ea7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:14,358 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:14,358 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., storeName=3f251ac37bffb103f032beac897bfe5d/C, priority=13, startTime=1731292333913; duration=0sec 2024-11-11T02:32:14,358 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:14,358 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f251ac37bffb103f032beac897bfe5d:C 2024-11-11T02:32:14,362 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/f1eb1a0e5f7340c78a5029c5ed9fd6e7 2024-11-11T02:32:14,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/f471e57a469b4d4a907fb850d9e650a5 is 50, key is test_row_0/B:col10/1731292332734/Put/seqid=0 2024-11-11T02:32:14,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742127_1303 (size=12301) 2024-11-11T02:32:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T02:32:14,775 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/f471e57a469b4d4a907fb850d9e650a5 2024-11-11T02:32:14,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/a86b6d28073c4ddeb77cfeb1b6b5b06f is 50, key is test_row_0/C:col10/1731292332734/Put/seqid=0 2024-11-11T02:32:14,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742128_1304 (size=12301) 2024-11-11T02:32:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:14,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. as already flushing 2024-11-11T02:32:14,877 INFO [Thread-1085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1085. 2024-11-11T02:32:14,877 DEBUG [Thread-1085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:14,877 DEBUG [Thread-1085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,877 DEBUG [Thread-1085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:14,908 INFO [Thread-1087 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1087. 2024-11-11T02:32:14,908 INFO [Thread-1091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1091. 2024-11-11T02:32:14,908 INFO [Thread-1089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1089. 2024-11-11T02:32:14,908 DEBUG [Thread-1089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:14,908 DEBUG [Thread-1091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:14,908 DEBUG [Thread-1087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:14,908 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,908 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,908 DEBUG [Thread-1087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,909 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,909 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,909 DEBUG [Thread-1087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:14,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:14,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:14,911 INFO [Thread-1093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1093. 2024-11-11T02:32:14,911 DEBUG [Thread-1093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:14,911 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,911 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:14,911 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:15,192 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/a86b6d28073c4ddeb77cfeb1b6b5b06f 2024-11-11T02:32:15,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/f1eb1a0e5f7340c78a5029c5ed9fd6e7 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/f1eb1a0e5f7340c78a5029c5ed9fd6e7 2024-11-11T02:32:15,207 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/f1eb1a0e5f7340c78a5029c5ed9fd6e7, entries=150, sequenceid=364, filesize=12.0 K 2024-11-11T02:32:15,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/f471e57a469b4d4a907fb850d9e650a5 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/f471e57a469b4d4a907fb850d9e650a5 2024-11-11T02:32:15,212 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/f471e57a469b4d4a907fb850d9e650a5, entries=150, sequenceid=364, filesize=12.0 K 2024-11-11T02:32:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/a86b6d28073c4ddeb77cfeb1b6b5b06f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a86b6d28073c4ddeb77cfeb1b6b5b06f 2024-11-11T02:32:15,215 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a86b6d28073c4ddeb77cfeb1b6b5b06f, entries=150, sequenceid=364, filesize=12.0 K 2024-11-11T02:32:15,216 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=33.54 KB/34350 for 3f251ac37bffb103f032beac897bfe5d in 1268ms, sequenceid=364, compaction requested=false 2024-11-11T02:32:15,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 3f251ac37bffb103f032beac897bfe5d: 2024-11-11T02:32:15,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:15,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-11T02:32:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-11T02:32:15,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-11T02:32:15,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8790 sec 2024-11-11T02:32:15,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.8830 sec 2024-11-11T02:32:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T02:32:15,487 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:15,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T02:32:15,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-11T02:32:15,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-11T02:32:15,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 131 2024-11-11T02:32:15,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-11T02:32:15,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-11-11T02:32:15,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T02:32:15,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3725 2024-11-11T02:32:15,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3237 2024-11-11T02:32:15,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3508 2024-11-11T02:32:15,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3750 2024-11-11T02:32:15,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3237 2024-11-11T02:32:15,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T02:32:15,489 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:32:15,489 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:32:15,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:15,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:15,490 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:15,491 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T02:32:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T02:32:15,499 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292335499"}]},"ts":"1731292335499"} 2024-11-11T02:32:15,501 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T02:32:15,501 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T02:32:15,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T02:32:15,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f251ac37bffb103f032beac897bfe5d, UNASSIGN}] 2024-11-11T02:32:15,506 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f251ac37bffb103f032beac897bfe5d, UNASSIGN 2024-11-11T02:32:15,507 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=3f251ac37bffb103f032beac897bfe5d, regionState=CLOSING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:32:15,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f251ac37bffb103f032beac897bfe5d, UNASSIGN because future has completed 2024-11-11T02:32:15,508 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T02:32:15,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:32:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T02:32:15,661 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:15,661 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T02:32:15,661 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 3f251ac37bffb103f032beac897bfe5d, disabling compactions & flushes 2024-11-11T02:32:15,662 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:15,662 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:15,662 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. after waiting 0 ms 2024-11-11T02:32:15,662 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:15,662 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing 3f251ac37bffb103f032beac897bfe5d 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T02:32:15,662 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=A 2024-11-11T02:32:15,663 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:15,663 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=B 2024-11-11T02:32:15,663 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:15,663 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f251ac37bffb103f032beac897bfe5d, store=C 2024-11-11T02:32:15,663 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:15,672 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/c86f8f68e1e749209008c290c19fc1d0 is 50, key is test_row_1/A:col10/1731292334908/Put/seqid=0 2024-11-11T02:32:15,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742129_1305 (size=9857) 2024-11-11T02:32:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T02:32:16,013 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T02:32:16,079 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/c86f8f68e1e749209008c290c19fc1d0 2024-11-11T02:32:16,093 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/16fb298407b14bee8bbc486b308b0118 is 50, key is test_row_1/B:col10/1731292334908/Put/seqid=0 2024-11-11T02:32:16,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742130_1306 (size=9857) 2024-11-11T02:32:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T02:32:16,500 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/16fb298407b14bee8bbc486b308b0118 2024-11-11T02:32:16,512 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/8da029d72fea4abcbe59d4e68cab8268 is 50, key is test_row_1/C:col10/1731292334908/Put/seqid=0 2024-11-11T02:32:16,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742131_1307 (size=9857) 2024-11-11T02:32:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T02:32:16,918 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/8da029d72fea4abcbe59d4e68cab8268 2024-11-11T02:32:16,928 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/A/c86f8f68e1e749209008c290c19fc1d0 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/c86f8f68e1e749209008c290c19fc1d0 2024-11-11T02:32:16,934 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/c86f8f68e1e749209008c290c19fc1d0, entries=100, sequenceid=374, filesize=9.6 K 2024-11-11T02:32:16,935 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/B/16fb298407b14bee8bbc486b308b0118 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/16fb298407b14bee8bbc486b308b0118 2024-11-11T02:32:16,939 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/16fb298407b14bee8bbc486b308b0118, entries=100, sequenceid=374, filesize=9.6 K 2024-11-11T02:32:16,939 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/.tmp/C/8da029d72fea4abcbe59d4e68cab8268 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8da029d72fea4abcbe59d4e68cab8268 2024-11-11T02:32:16,943 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8da029d72fea4abcbe59d4e68cab8268, entries=100, sequenceid=374, filesize=9.6 K 2024-11-11T02:32:16,944 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3f251ac37bffb103f032beac897bfe5d in 1282ms, sequenceid=374, compaction requested=true 2024-11-11T02:32:16,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/3dd1df037fe947fea24232ac5d85788e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/06fe342b8db142beb227031e8fb7baa4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/50e784ef72074e739a0e73b0f6f1fddf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/4df0a5f8291449c3a87e825d017b374e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/93bce23e0e044e5491b91ef25d9d9158, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/48aacb08a91d4f1aafb560f21b905ce1, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/dc426f62c8034bdba97bdafc08e4a05a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/6ccedf6bc61344f2b7b825bc34c4703e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/a10396b53f534737960c11f404ec31cb, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/01235c647ca1423fb63e970439a0a489, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/35ac6d0d2c3448bdb4231d62059d7acf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/93e4076ec63f4fad9fcf4dbf01f875a0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/f8f945956d0949cbb15a6c84ef32219c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/595b52a88057475e889b3af6efc90e9f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/1b5ce95b013a4e21afacac177257233c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/1796f1abb0934fb4b5936dae65f47efa, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b60c963c16e04c3096c21ec00598eb0e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/77a3e35e28cc469d8c42fde29003b57d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b1bbae3edcd34f67890de8c442dc918b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/7995f781aa2349319d9e758b2ca20a72, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/eb78233ae24646eaabab8dd65dfc92fc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b00022da1fb54f12a0ab10b2e3b029d7] to archive 2024-11-11T02:32:16,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:32:16,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/3dd1df037fe947fea24232ac5d85788e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/3dd1df037fe947fea24232ac5d85788e 2024-11-11T02:32:16,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/06fe342b8db142beb227031e8fb7baa4 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/06fe342b8db142beb227031e8fb7baa4 2024-11-11T02:32:16,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/50e784ef72074e739a0e73b0f6f1fddf to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/50e784ef72074e739a0e73b0f6f1fddf 2024-11-11T02:32:16,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/4df0a5f8291449c3a87e825d017b374e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/4df0a5f8291449c3a87e825d017b374e 2024-11-11T02:32:16,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/93bce23e0e044e5491b91ef25d9d9158 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/93bce23e0e044e5491b91ef25d9d9158 2024-11-11T02:32:16,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/48aacb08a91d4f1aafb560f21b905ce1 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/48aacb08a91d4f1aafb560f21b905ce1 2024-11-11T02:32:16,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/dc426f62c8034bdba97bdafc08e4a05a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/dc426f62c8034bdba97bdafc08e4a05a 2024-11-11T02:32:16,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/6ccedf6bc61344f2b7b825bc34c4703e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/6ccedf6bc61344f2b7b825bc34c4703e 2024-11-11T02:32:16,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/a10396b53f534737960c11f404ec31cb to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/a10396b53f534737960c11f404ec31cb 2024-11-11T02:32:16,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/01235c647ca1423fb63e970439a0a489 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/01235c647ca1423fb63e970439a0a489 2024-11-11T02:32:16,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/35ac6d0d2c3448bdb4231d62059d7acf to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/35ac6d0d2c3448bdb4231d62059d7acf 2024-11-11T02:32:16,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/93e4076ec63f4fad9fcf4dbf01f875a0 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/93e4076ec63f4fad9fcf4dbf01f875a0 2024-11-11T02:32:16,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/f8f945956d0949cbb15a6c84ef32219c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/f8f945956d0949cbb15a6c84ef32219c 2024-11-11T02:32:16,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/595b52a88057475e889b3af6efc90e9f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/595b52a88057475e889b3af6efc90e9f 2024-11-11T02:32:16,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/1b5ce95b013a4e21afacac177257233c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/1b5ce95b013a4e21afacac177257233c 2024-11-11T02:32:16,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/1796f1abb0934fb4b5936dae65f47efa to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/1796f1abb0934fb4b5936dae65f47efa 2024-11-11T02:32:16,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b60c963c16e04c3096c21ec00598eb0e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b60c963c16e04c3096c21ec00598eb0e 2024-11-11T02:32:16,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/77a3e35e28cc469d8c42fde29003b57d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/77a3e35e28cc469d8c42fde29003b57d 2024-11-11T02:32:16,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b1bbae3edcd34f67890de8c442dc918b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b1bbae3edcd34f67890de8c442dc918b 2024-11-11T02:32:16,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/7995f781aa2349319d9e758b2ca20a72 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/7995f781aa2349319d9e758b2ca20a72 2024-11-11T02:32:16,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/eb78233ae24646eaabab8dd65dfc92fc to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/eb78233ae24646eaabab8dd65dfc92fc 2024-11-11T02:32:16,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b00022da1fb54f12a0ab10b2e3b029d7 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/b00022da1fb54f12a0ab10b2e3b029d7 2024-11-11T02:32:16,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/99c702fcf85045a686b2ced9372d479c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/b351be31e0e442d8b267fcc14db4daa8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/98655c671ebf4ce0811cf027f522bdce, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/3cccee7e35244fe3879399f20ffd02c6, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/17689d5f56fc413ea0fe1549fac6a05b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/489b4e7869474f5dbd1ed8b66371ecdb, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/8ee882b6afa8448eb0cdea4eacf47a52, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/7275f8876a69447087beafef15a73318, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/8ed0ac7691d7490791462808641aabdf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/ba456d3e2cdc40f7b3ff7f5b7b4be5fa, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/67dee509dc624f66b9c7cb037588bfbc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/c2d24129f1834bc88f58e822c437f254, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/5446155d5c6a4317937224d3173e6d57, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/bb30f5a899f049609678630c20ef1e77, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/c872f95896014bd3986147d9c81840c0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/6963816c671b4f438f1f384d81e36a5b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/483c0576058448ad8d22cb8fd6d133af, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/45ffd8c026b04121ba5ceaeb27c3a01e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/b46c32b881544b3195a59d225bc383e0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/457cf1890ab8491faf879915cc4e66ca, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/57d49f869e264459b08c604408fd1a22, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/6258d816df2b4895a3702ca9dba194a5] to archive 2024-11-11T02:32:16,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:32:16,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/99c702fcf85045a686b2ced9372d479c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/99c702fcf85045a686b2ced9372d479c 2024-11-11T02:32:16,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/b351be31e0e442d8b267fcc14db4daa8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/b351be31e0e442d8b267fcc14db4daa8 2024-11-11T02:32:16,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/98655c671ebf4ce0811cf027f522bdce to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/98655c671ebf4ce0811cf027f522bdce 2024-11-11T02:32:16,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/3cccee7e35244fe3879399f20ffd02c6 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/3cccee7e35244fe3879399f20ffd02c6 2024-11-11T02:32:16,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/17689d5f56fc413ea0fe1549fac6a05b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/17689d5f56fc413ea0fe1549fac6a05b 2024-11-11T02:32:16,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/489b4e7869474f5dbd1ed8b66371ecdb to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/489b4e7869474f5dbd1ed8b66371ecdb 2024-11-11T02:32:16,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/8ee882b6afa8448eb0cdea4eacf47a52 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/8ee882b6afa8448eb0cdea4eacf47a52 2024-11-11T02:32:16,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/7275f8876a69447087beafef15a73318 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/7275f8876a69447087beafef15a73318 2024-11-11T02:32:16,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/8ed0ac7691d7490791462808641aabdf to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/8ed0ac7691d7490791462808641aabdf 2024-11-11T02:32:16,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/ba456d3e2cdc40f7b3ff7f5b7b4be5fa to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/ba456d3e2cdc40f7b3ff7f5b7b4be5fa 2024-11-11T02:32:16,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/67dee509dc624f66b9c7cb037588bfbc to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/67dee509dc624f66b9c7cb037588bfbc 2024-11-11T02:32:16,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/c2d24129f1834bc88f58e822c437f254 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/c2d24129f1834bc88f58e822c437f254 2024-11-11T02:32:16,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/5446155d5c6a4317937224d3173e6d57 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/5446155d5c6a4317937224d3173e6d57 2024-11-11T02:32:16,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/bb30f5a899f049609678630c20ef1e77 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/bb30f5a899f049609678630c20ef1e77 2024-11-11T02:32:16,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/c872f95896014bd3986147d9c81840c0 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/c872f95896014bd3986147d9c81840c0 2024-11-11T02:32:16,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/6963816c671b4f438f1f384d81e36a5b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/6963816c671b4f438f1f384d81e36a5b 2024-11-11T02:32:16,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/483c0576058448ad8d22cb8fd6d133af to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/483c0576058448ad8d22cb8fd6d133af 2024-11-11T02:32:16,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/45ffd8c026b04121ba5ceaeb27c3a01e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/45ffd8c026b04121ba5ceaeb27c3a01e 2024-11-11T02:32:16,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/b46c32b881544b3195a59d225bc383e0 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/b46c32b881544b3195a59d225bc383e0 2024-11-11T02:32:16,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/457cf1890ab8491faf879915cc4e66ca to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/457cf1890ab8491faf879915cc4e66ca 2024-11-11T02:32:16,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/57d49f869e264459b08c604408fd1a22 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/57d49f869e264459b08c604408fd1a22 2024-11-11T02:32:16,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/6258d816df2b4895a3702ca9dba194a5 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/6258d816df2b4895a3702ca9dba194a5 2024-11-11T02:32:16,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/29acc1e31b4b4f43ac61b95e1d8ec855, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/fc480f9fb4164776a84388432cbf12b1, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/47c719b86c3f4df3b6a01e8d5095ce23, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/78684e78c1b740938ff50bd6d1f709df, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/c583220ac66c48e88ab73795096445e2, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8d2826ea829f47c18129a27fdf749bb4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/1f1134d136684a62add1167f70ede5ee, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/5eaf09d21f9d412694b4bdda17d2163f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/84e69a41643d49b493989adf8c381e6a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8885cc2487634b00973d0fe12fee18c8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/c6236809d3134883b399784bf0c0b83e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/31d45b43804142b4be10ceb255cee927, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/108f1e8b386e42199b35ba0b3bcecc5e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/688a0172f9dd4f98be672af8fac5bb6d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a59b3f300be144f2a4e92bacd1f3db18, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/944be338acb149b29f038a5880816955, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a56bdc6ef2e441059dce6ebc12c02a7d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a714a89d0ef6411385768c589d4ea107, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/1455cffc6cec426ea1fb11a47caeed78, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/19de398057a1480dbbc552d862ac5722, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/6585b48cf0a34817a89e5fde82d77913, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/0263f6da882643309de7b421d2be8b9a] to archive 2024-11-11T02:32:16,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:32:16,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/29acc1e31b4b4f43ac61b95e1d8ec855 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/29acc1e31b4b4f43ac61b95e1d8ec855 2024-11-11T02:32:16,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/fc480f9fb4164776a84388432cbf12b1 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/fc480f9fb4164776a84388432cbf12b1 2024-11-11T02:32:16,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/47c719b86c3f4df3b6a01e8d5095ce23 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/47c719b86c3f4df3b6a01e8d5095ce23 2024-11-11T02:32:16,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/78684e78c1b740938ff50bd6d1f709df to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/78684e78c1b740938ff50bd6d1f709df 2024-11-11T02:32:16,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/c583220ac66c48e88ab73795096445e2 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/c583220ac66c48e88ab73795096445e2 2024-11-11T02:32:16,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8d2826ea829f47c18129a27fdf749bb4 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8d2826ea829f47c18129a27fdf749bb4 2024-11-11T02:32:16,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/1f1134d136684a62add1167f70ede5ee to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/1f1134d136684a62add1167f70ede5ee 2024-11-11T02:32:16,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/5eaf09d21f9d412694b4bdda17d2163f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/5eaf09d21f9d412694b4bdda17d2163f 2024-11-11T02:32:16,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/84e69a41643d49b493989adf8c381e6a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/84e69a41643d49b493989adf8c381e6a 2024-11-11T02:32:16,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8885cc2487634b00973d0fe12fee18c8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8885cc2487634b00973d0fe12fee18c8 2024-11-11T02:32:16,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/c6236809d3134883b399784bf0c0b83e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/c6236809d3134883b399784bf0c0b83e 2024-11-11T02:32:16,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/31d45b43804142b4be10ceb255cee927 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/31d45b43804142b4be10ceb255cee927 2024-11-11T02:32:16,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/108f1e8b386e42199b35ba0b3bcecc5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/108f1e8b386e42199b35ba0b3bcecc5e 2024-11-11T02:32:16,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/688a0172f9dd4f98be672af8fac5bb6d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/688a0172f9dd4f98be672af8fac5bb6d 2024-11-11T02:32:16,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a59b3f300be144f2a4e92bacd1f3db18 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a59b3f300be144f2a4e92bacd1f3db18 2024-11-11T02:32:16,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/944be338acb149b29f038a5880816955 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/944be338acb149b29f038a5880816955 2024-11-11T02:32:17,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a56bdc6ef2e441059dce6ebc12c02a7d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a56bdc6ef2e441059dce6ebc12c02a7d 2024-11-11T02:32:17,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a714a89d0ef6411385768c589d4ea107 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a714a89d0ef6411385768c589d4ea107 2024-11-11T02:32:17,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/1455cffc6cec426ea1fb11a47caeed78 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/1455cffc6cec426ea1fb11a47caeed78 2024-11-11T02:32:17,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/19de398057a1480dbbc552d862ac5722 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/19de398057a1480dbbc552d862ac5722 2024-11-11T02:32:17,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/6585b48cf0a34817a89e5fde82d77913 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/6585b48cf0a34817a89e5fde82d77913 2024-11-11T02:32:17,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/0263f6da882643309de7b421d2be8b9a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/0263f6da882643309de7b421d2be8b9a 2024-11-11T02:32:17,007 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/recovered.edits/377.seqid, newMaxSeqId=377, maxSeqId=1 2024-11-11T02:32:17,007 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. 2024-11-11T02:32:17,007 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 3f251ac37bffb103f032beac897bfe5d: Waiting for close lock at 1731292335661Running coprocessor pre-close hooks at 1731292335661Disabling compacts and flushes for region at 1731292335661Disabling writes for close at 1731292335662 (+1 ms)Obtaining lock to block concurrent updates at 1731292335662Preparing flush snapshotting stores in 3f251ac37bffb103f032beac897bfe5d at 1731292335662Finished memstore snapshotting TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731292335663 (+1 ms)Flushing stores of TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d. at 1731292335664 (+1 ms)Flushing 3f251ac37bffb103f032beac897bfe5d/A: creating writer at 1731292335664Flushing 3f251ac37bffb103f032beac897bfe5d/A: appending metadata at 1731292335671 (+7 ms)Flushing 3f251ac37bffb103f032beac897bfe5d/A: closing flushed file at 1731292335671Flushing 3f251ac37bffb103f032beac897bfe5d/B: creating writer at 1731292336087 (+416 ms)Flushing 3f251ac37bffb103f032beac897bfe5d/B: appending metadata at 1731292336092 (+5 ms)Flushing 3f251ac37bffb103f032beac897bfe5d/B: closing flushed file at 1731292336092Flushing 3f251ac37bffb103f032beac897bfe5d/C: creating writer at 1731292336509 (+417 ms)Flushing 3f251ac37bffb103f032beac897bfe5d/C: appending metadata at 1731292336512 (+3 ms)Flushing 3f251ac37bffb103f032beac897bfe5d/C: closing flushed file at 1731292336512Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e40bc7e: reopening flushed file at 1731292336927 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ebbfcff: reopening flushed file at 1731292336934 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31f9b8ee: reopening flushed file at 1731292336939 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3f251ac37bffb103f032beac897bfe5d in 1282ms, sequenceid=374, compaction requested=true at 1731292336944 (+5 ms)Writing region close event to WAL at 1731292337005 (+61 ms)Running coprocessor post-close hooks at 1731292337007 (+2 ms)Closed at 1731292337007 2024-11-11T02:32:17,009 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:17,009 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=3f251ac37bffb103f032beac897bfe5d, regionState=CLOSED 2024-11-11T02:32:17,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:32:17,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-11T02:32:17,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 3f251ac37bffb103f032beac897bfe5d, server=370bc2ade342,38037,1731292247443 in 1.5030 sec 2024-11-11T02:32:17,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-11T02:32:17,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f251ac37bffb103f032beac897bfe5d, UNASSIGN in 1.5080 sec 2024-11-11T02:32:17,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-11T02:32:17,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5120 sec 2024-11-11T02:32:17,018 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292337018"}]},"ts":"1731292337018"} 2024-11-11T02:32:17,020 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T02:32:17,020 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T02:32:17,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5300 sec 2024-11-11T02:32:17,210 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T02:32:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T02:32:17,657 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:17,659 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T02:32:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:17,664 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T02:32:17,666 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:17,668 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:17,672 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/recovered.edits] 2024-11-11T02:32:17,675 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/7cdf4c7e56c346908498b9db342d07da to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/7cdf4c7e56c346908498b9db342d07da 2024-11-11T02:32:17,677 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/c86f8f68e1e749209008c290c19fc1d0 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/c86f8f68e1e749209008c290c19fc1d0 2024-11-11T02:32:17,679 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/f1eb1a0e5f7340c78a5029c5ed9fd6e7 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/A/f1eb1a0e5f7340c78a5029c5ed9fd6e7 2024-11-11T02:32:17,683 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/16fb298407b14bee8bbc486b308b0118 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/16fb298407b14bee8bbc486b308b0118 2024-11-11T02:32:17,685 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/f471e57a469b4d4a907fb850d9e650a5 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/f471e57a469b4d4a907fb850d9e650a5 2024-11-11T02:32:17,688 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/f51ccf69211d4583af35533e37ca84b6 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/B/f51ccf69211d4583af35533e37ca84b6 2024-11-11T02:32:17,692 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8da029d72fea4abcbe59d4e68cab8268 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/8da029d72fea4abcbe59d4e68cab8268 2024-11-11T02:32:17,694 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a86b6d28073c4ddeb77cfeb1b6b5b06f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/a86b6d28073c4ddeb77cfeb1b6b5b06f 2024-11-11T02:32:17,696 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/bb9da215dbf64dceac7f6dacd39b2ea7 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/C/bb9da215dbf64dceac7f6dacd39b2ea7 2024-11-11T02:32:17,701 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/recovered.edits/377.seqid to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d/recovered.edits/377.seqid 2024-11-11T02:32:17,702 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/3f251ac37bffb103f032beac897bfe5d 2024-11-11T02:32:17,702 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T02:32:17,705 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:17,710 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T02:32:17,713 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T02:32:17,714 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:17,714 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T02:32:17,715 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731292337714"}]},"ts":"9223372036854775807"} 2024-11-11T02:32:17,717 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T02:32:17,717 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3f251ac37bffb103f032beac897bfe5d, NAME => 'TestAcidGuarantees,,1731292312035.3f251ac37bffb103f032beac897bfe5d.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T02:32:17,717 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T02:32:17,718 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731292337717"}]},"ts":"9223372036854775807"} 2024-11-11T02:32:17,720 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T02:32:17,721 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:17,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-11T02:32:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T02:32:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T02:32:17,777 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:17,791 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249 (was 248) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2002917628_22 at /127.0.0.1:45030 [Waiting for operation #468] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=623 (was 744), ProcessCount=11 (was 11), AvailableMemoryMB=5618 (was 5675) 2024-11-11T02:32:17,802 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=623, ProcessCount=11, AvailableMemoryMB=5618 2024-11-11T02:32:17,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T02:32:17,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:32:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:17,805 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T02:32:17,805 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:17,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-11T02:32:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T02:32:17,806 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T02:32:17,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742132_1308 (size=1061) 2024-11-11T02:32:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T02:32:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T02:32:18,216 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe 2024-11-11T02:32:18,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742133_1309 (size=53) 2024-11-11T02:32:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T02:32:18,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:32:18,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 875c7d02961ba5720596e10e2c5e2c5e, disabling compactions & flushes 2024-11-11T02:32:18,628 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:18,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:18,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. after waiting 0 ms 2024-11-11T02:32:18,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:18,628 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:18,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 875c7d02961ba5720596e10e2c5e2c5e: Waiting for close lock at 1731292338628Disabling compacts and flushes for region at 1731292338628Disabling writes for close at 1731292338628Writing region close event to WAL at 1731292338628Closed at 1731292338628 2024-11-11T02:32:18,630 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T02:32:18,630 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731292338630"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731292338630"}]},"ts":"1731292338630"} 2024-11-11T02:32:18,632 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T02:32:18,633 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T02:32:18,633 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292338633"}]},"ts":"1731292338633"} 2024-11-11T02:32:18,635 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T02:32:18,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, ASSIGN}] 2024-11-11T02:32:18,637 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, ASSIGN 2024-11-11T02:32:18,638 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, ASSIGN; state=OFFLINE, location=370bc2ade342,38037,1731292247443; forceNewPlan=false, retain=false 2024-11-11T02:32:18,790 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=875c7d02961ba5720596e10e2c5e2c5e, regionState=OPENING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:32:18,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, ASSIGN because future has completed 2024-11-11T02:32:18,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:32:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T02:32:18,952 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:18,952 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:32:18,952 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:18,952 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:32:18,952 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:18,952 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:18,954 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:18,955 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:32:18,955 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 875c7d02961ba5720596e10e2c5e2c5e columnFamilyName A 2024-11-11T02:32:18,956 DEBUG [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:18,956 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.HStore(327): Store=875c7d02961ba5720596e10e2c5e2c5e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:32:18,956 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:18,957 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:32:18,958 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 875c7d02961ba5720596e10e2c5e2c5e columnFamilyName B 2024-11-11T02:32:18,958 DEBUG [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:18,958 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.HStore(327): Store=875c7d02961ba5720596e10e2c5e2c5e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:32:18,958 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:18,959 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:32:18,960 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 875c7d02961ba5720596e10e2c5e2c5e columnFamilyName C 2024-11-11T02:32:18,960 DEBUG [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:18,960 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.HStore(327): Store=875c7d02961ba5720596e10e2c5e2c5e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:32:18,960 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:18,960 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:18,961 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:18,962 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:18,963 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:18,963 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:18,964 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:32:18,965 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:18,968 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:32:18,968 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 875c7d02961ba5720596e10e2c5e2c5e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75403828, jitterRate=0.12360459566116333}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:32:18,969 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:18,970 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 875c7d02961ba5720596e10e2c5e2c5e: Running coprocessor pre-open hook at 1731292338952Writing region info on filesystem at 1731292338952Initializing all the Stores at 1731292338953 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292338953Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292338954 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292338954Cleaning up temporary data from old regions at 1731292338963 (+9 ms)Running coprocessor post-open hooks at 1731292338969 (+6 ms)Region opened successfully at 1731292338970 (+1 ms) 2024-11-11T02:32:18,971 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., pid=91, masterSystemTime=1731292338949 2024-11-11T02:32:18,973 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:18,973 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:18,974 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=875c7d02961ba5720596e10e2c5e2c5e, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:32:18,975 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42785 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=370bc2ade342,38037,1731292247443, table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T02:32:18,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:32:18,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-11T02:32:18,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 in 182 msec 2024-11-11T02:32:18,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-11T02:32:18,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, ASSIGN in 343 msec 2024-11-11T02:32:18,982 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T02:32:18,982 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292338982"}]},"ts":"1731292338982"} 2024-11-11T02:32:18,983 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T02:32:18,984 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T02:32:18,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-11-11T02:32:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T02:32:19,957 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:19,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f9102f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:19,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:19,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:19,960 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:19,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663b188, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:19,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:19,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:19,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:19,963 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:19,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b5cf238, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:19,965 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:19,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:19,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:19,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:19,969 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T02:32:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 370bc2ade342,42785,1731292246873 2024-11-11T02:32:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@48f3e1d0 2024-11-11T02:32:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T02:32:19,971 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T02:32:19,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T02:32:19,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:32:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:19,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742134_1310 (size=1097) 2024-11-11T02:32:20,388 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T02:32:20,388 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T02:32:20,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T02:32:20,395 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, REOPEN/MOVE}] 2024-11-11T02:32:20,396 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, REOPEN/MOVE 2024-11-11T02:32:20,397 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=875c7d02961ba5720596e10e2c5e2c5e, regionState=CLOSING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:32:20,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, REOPEN/MOVE because future has completed 2024-11-11T02:32:20,398 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T02:32:20,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:32:20,551 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,551 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T02:32:20,551 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 875c7d02961ba5720596e10e2c5e2c5e, disabling compactions & flushes 2024-11-11T02:32:20,551 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:20,551 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:20,551 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. after waiting 0 ms 2024-11-11T02:32:20,552 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:20,558 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T02:32:20,560 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:20,560 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 875c7d02961ba5720596e10e2c5e2c5e: Waiting for close lock at 1731292340551Running coprocessor pre-close hooks at 1731292340551Disabling compacts and flushes for region at 1731292340551Disabling writes for close at 1731292340552 (+1 ms)Writing region close event to WAL at 1731292340552Running coprocessor post-close hooks at 1731292340559 (+7 ms)Closed at 1731292340559 2024-11-11T02:32:20,560 WARN [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 875c7d02961ba5720596e10e2c5e2c5e to self. 2024-11-11T02:32:20,563 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,565 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=875c7d02961ba5720596e10e2c5e2c5e, regionState=CLOSED 2024-11-11T02:32:20,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:32:20,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-11T02:32:20,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 in 170 msec 2024-11-11T02:32:20,572 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, REOPEN/MOVE; state=CLOSED, location=370bc2ade342,38037,1731292247443; forceNewPlan=false, retain=true 2024-11-11T02:32:20,723 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=875c7d02961ba5720596e10e2c5e2c5e, regionState=OPENING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:32:20,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, REOPEN/MOVE because future has completed 2024-11-11T02:32:20,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:32:20,891 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:20,892 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:32:20,892 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,893 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:32:20,893 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,893 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,897 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,898 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:32:20,898 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 875c7d02961ba5720596e10e2c5e2c5e columnFamilyName A 2024-11-11T02:32:20,908 DEBUG [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:20,908 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.HStore(327): Store=875c7d02961ba5720596e10e2c5e2c5e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:32:20,909 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,909 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:32:20,909 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 875c7d02961ba5720596e10e2c5e2c5e columnFamilyName B 2024-11-11T02:32:20,909 DEBUG [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:20,910 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.HStore(327): Store=875c7d02961ba5720596e10e2c5e2c5e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:32:20,910 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,910 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:32:20,910 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 875c7d02961ba5720596e10e2c5e2c5e columnFamilyName C 2024-11-11T02:32:20,910 DEBUG [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:20,911 INFO [StoreOpener-875c7d02961ba5720596e10e2c5e2c5e-1 {}] regionserver.HStore(327): Store=875c7d02961ba5720596e10e2c5e2c5e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:32:20,911 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:20,911 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,911 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,912 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,913 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,913 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,913 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:32:20,914 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,915 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 875c7d02961ba5720596e10e2c5e2c5e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70652602, jitterRate=0.052805811166763306}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:32:20,915 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,915 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 875c7d02961ba5720596e10e2c5e2c5e: Running coprocessor pre-open hook at 1731292340893Writing region info on filesystem at 1731292340893Initializing all the Stores at 1731292340895 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292340896 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292340896Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292340896Cleaning up temporary data from old regions at 1731292340913 (+17 ms)Running coprocessor post-open hooks at 1731292340915 (+2 ms)Region opened successfully at 1731292340915 2024-11-11T02:32:20,916 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., pid=96, masterSystemTime=1731292340883 2024-11-11T02:32:20,918 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:20,918 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:20,918 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=875c7d02961ba5720596e10e2c5e2c5e, regionState=OPEN, openSeqNum=5, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:32:20,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:32:20,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-11T02:32:20,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 in 192 msec 2024-11-11T02:32:20,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-11T02:32:20,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, REOPEN/MOVE in 527 msec 2024-11-11T02:32:20,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-11T02:32:20,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-11-11T02:32:20,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 952 msec 2024-11-11T02:32:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-11T02:32:20,929 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:20,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec1958f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,929 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:20,930 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:20,930 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195b0a10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:20,932 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35780b0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:20,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:20,934 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7232271e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:20,936 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56df809e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:20,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:20,938 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:20,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6210764e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:20,940 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:20,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fb5b1f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:20,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:20,942 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e98bcfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:20,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:20,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:20,943 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:20,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b305395, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:20,944 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:20,945 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38e0f32b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:20,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:20,946 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:20,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c75301f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:20,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:20,948 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:20,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:20,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:20,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69c4c5bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:20,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:20,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:20,949 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:20,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@110712fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,950 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:20,950 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:20,951 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:20,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:20,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:20,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7324d2c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:20,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:20,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:20,952 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24075f5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:20,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:20,954 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0ad8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:20,956 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4493539, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:20,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:20,958 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b812c98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:20,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:20,959 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311b8876, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:20,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:20,962 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@200095ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:20,963 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:20,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25ec8432, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:20,965 DEBUG [Thread-1441 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:20,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:20,965 DEBUG [Thread-1428 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:20,965 DEBUG [Thread-1435 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:20,965 DEBUG [Thread-1443 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:20,965 DEBUG [Thread-1439 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:20,966 DEBUG [Thread-1437 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:20,966 DEBUG [Thread-1432 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:20,967 DEBUG [Thread-1426 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:20,968 DEBUG [Thread-1424 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:20,968 DEBUG [Thread-1430 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:20,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:20,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:20,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:20,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:20,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:20,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:20,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-11T02:32:20,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:20,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:20,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:32:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:32:20,973 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T02:32:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:32:20,974 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:20,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:32:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:32:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:32:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:32:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:20,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:32:20,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:32:20,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:32:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:20,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:32:20,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:20,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:20,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:20,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:20,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:20,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:21,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d9f5473adce8496f8e14937fab9965c0_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292340987/Put/seqid=0 2024-11-11T02:32:21,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742135_1311 (size=14594) 2024-11-11T02:32:21,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292341000, max timestamp=1731292341028Num uniques 859; 2024-11-11T02:32:21,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:21,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292341000, max timestamp=1731292341028Num uniques 859; 2024-11-11T02:32:21,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:21,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292341000, max timestamp=1731292341028Num uniques 859; 2024-11-11T02:32:21,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:32:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T02:32:21,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292401067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292401067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292401068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292401068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731292401069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T02:32:21,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:21,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:21,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:21,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292401186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292401186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292401186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292401186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731292401196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T02:32:21,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:21,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:21,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:21,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T02:32:21,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292401405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292401405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292401405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292401406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,424 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:21,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731292401416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,428 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d9f5473adce8496f8e14937fab9965c0_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d9f5473adce8496f8e14937fab9965c0_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:21,429 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/ec6101a136924812a6c3a037d88091fa, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:21,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/ec6101a136924812a6c3a037d88091fa is 175, key is test_row_0/A:col10/1731292340987/Put/seqid=0 2024-11-11T02:32:21,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T02:32:21,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:21,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:21,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:21,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742136_1312 (size=39549) 2024-11-11T02:32:21,439 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/ec6101a136924812a6c3a037d88091fa 2024-11-11T02:32:21,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/5657c53483a84ecf83a065051cea565e is 50, key is test_row_0/B:col10/1731292340987/Put/seqid=0 2024-11-11T02:32:21,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742137_1313 (size=12001) 2024-11-11T02:32:21,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/5657c53483a84ecf83a065051cea565e 2024-11-11T02:32:21,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/49f952f6b34f4267a770b23460c94069 is 50, key is test_row_0/C:col10/1731292340987/Put/seqid=0 2024-11-11T02:32:21,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742138_1314 (size=12001) 2024-11-11T02:32:21,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T02:32:21,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:21,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:21,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:21,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T02:32:21,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292401726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292401726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292401726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292401726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T02:32:21,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:21,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:21,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:21,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731292401735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:21,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T02:32:21,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:21,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:21,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:21,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/49f952f6b34f4267a770b23460c94069 2024-11-11T02:32:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/ec6101a136924812a6c3a037d88091fa as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/ec6101a136924812a6c3a037d88091fa 2024-11-11T02:32:21,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/ec6101a136924812a6c3a037d88091fa, entries=200, sequenceid=15, filesize=38.6 K 2024-11-11T02:32:21,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/5657c53483a84ecf83a065051cea565e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/5657c53483a84ecf83a065051cea565e 2024-11-11T02:32:21,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/5657c53483a84ecf83a065051cea565e, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T02:32:21,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/49f952f6b34f4267a770b23460c94069 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/49f952f6b34f4267a770b23460c94069 2024-11-11T02:32:21,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/49f952f6b34f4267a770b23460c94069, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T02:32:21,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 875c7d02961ba5720596e10e2c5e2c5e in 955ms, sequenceid=15, compaction requested=false 2024-11-11T02:32:21,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:22,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T02:32:22,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:22,047 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-11-11T02:32:22,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:22,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:22,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:22,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:22,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:22,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ed9f4e5d8bbb4b48a8f9a9a4ebbad4ef_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292341068/Put/seqid=0 2024-11-11T02:32:22,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742139_1315 (size=12154) 2024-11-11T02:32:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T02:32:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:22,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:22,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292402254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292402255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731292402256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292402257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292402266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292402376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292402376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292402376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292402376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:22,466 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ed9f4e5d8bbb4b48a8f9a9a4ebbad4ef_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ed9f4e5d8bbb4b48a8f9a9a4ebbad4ef_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/cb397d7505394396866b4700b3c4855b, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:22,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/cb397d7505394396866b4700b3c4855b is 175, key is test_row_0/A:col10/1731292341068/Put/seqid=0 2024-11-11T02:32:22,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742140_1316 (size=30955) 2024-11-11T02:32:22,545 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T02:32:22,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292402595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292402595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292402595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292402595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,873 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/cb397d7505394396866b4700b3c4855b 2024-11-11T02:32:22,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/8217b7328b284ed79a8debdcc0772995 is 50, key is test_row_0/B:col10/1731292341068/Put/seqid=0 2024-11-11T02:32:22,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742141_1317 (size=12001) 2024-11-11T02:32:22,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292402916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292402917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292402917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292402917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T02:32:23,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731292403276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:23,284 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in 875c7d02961ba5720596e10e2c5e2c5e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:32:23,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:23,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:23,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:23,304 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/8217b7328b284ed79a8debdcc0772995 2024-11-11T02:32:23,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/b1cf6baa2d7641ff9090d652caa2073c is 50, key is test_row_0/C:col10/1731292341068/Put/seqid=0 2024-11-11T02:32:23,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742142_1318 (size=12001) 2024-11-11T02:32:23,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292403435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:23,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292403435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:23,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292403435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:23,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292403435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:23,715 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/b1cf6baa2d7641ff9090d652caa2073c 2024-11-11T02:32:23,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/cb397d7505394396866b4700b3c4855b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/cb397d7505394396866b4700b3c4855b 2024-11-11T02:32:23,721 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/cb397d7505394396866b4700b3c4855b, entries=150, sequenceid=50, filesize=30.2 K 2024-11-11T02:32:23,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/8217b7328b284ed79a8debdcc0772995 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/8217b7328b284ed79a8debdcc0772995 2024-11-11T02:32:23,726 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/8217b7328b284ed79a8debdcc0772995, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T02:32:23,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/b1cf6baa2d7641ff9090d652caa2073c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/b1cf6baa2d7641ff9090d652caa2073c 2024-11-11T02:32:23,731 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/b1cf6baa2d7641ff9090d652caa2073c, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T02:32:23,732 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=53.67 KB/54960 for 875c7d02961ba5720596e10e2c5e2c5e in 1685ms, sequenceid=50, compaction requested=false 2024-11-11T02:32:23,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:23,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:23,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-11T02:32:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-11T02:32:23,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-11T02:32:23,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7590 sec 2024-11-11T02:32:23,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.7700 sec 2024-11-11T02:32:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:24,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T02:32:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:24,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e69e878ce4aa403a86719cdb8edf38c6_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292342265/Put/seqid=0 2024-11-11T02:32:24,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742143_1319 (size=14594) 2024-11-11T02:32:24,470 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:24,473 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e69e878ce4aa403a86719cdb8edf38c6_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e69e878ce4aa403a86719cdb8edf38c6_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:24,474 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/cc01b7591c16424aa1adcaf8123f2b66, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:24,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/cc01b7591c16424aa1adcaf8123f2b66 is 175, key is test_row_0/A:col10/1731292342265/Put/seqid=0 2024-11-11T02:32:24,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742144_1320 (size=39549) 2024-11-11T02:32:24,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292404530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:24,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:24,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292404531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:24,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:24,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292404533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:24,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:24,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292404543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:24,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292404646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:24,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292404646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:24,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292404656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:24,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292404656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:24,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292404855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:24,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292404855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:24,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292404865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:24,882 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/cc01b7591c16424aa1adcaf8123f2b66 2024-11-11T02:32:24,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292404875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:24,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/62c867129a734878aebe78446b657ef6 is 50, key is test_row_0/B:col10/1731292342265/Put/seqid=0 2024-11-11T02:32:24,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742145_1321 (size=12001) 2024-11-11T02:32:24,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/62c867129a734878aebe78446b657ef6 2024-11-11T02:32:24,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/1a7f8dc4bb2b433898a4d87a1e1a1255 is 50, key is test_row_0/C:col10/1731292342265/Put/seqid=0 2024-11-11T02:32:24,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742146_1322 (size=12001) 2024-11-11T02:32:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T02:32:25,136 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:25,136 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-11T02:32:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T02:32:25,138 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:25,138 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:25,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:25,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292405176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:25,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292405176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:25,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292405185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:25,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292405195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:25,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T02:32:25,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T02:32:25,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:25,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:25,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:25,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:25,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:25,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:25,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:25,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731292405295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:25,305 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in 875c7d02961ba5720596e10e2c5e2c5e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:32:25,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:25,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/1a7f8dc4bb2b433898a4d87a1e1a1255 2024-11-11T02:32:25,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/cc01b7591c16424aa1adcaf8123f2b66 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/cc01b7591c16424aa1adcaf8123f2b66 2024-11-11T02:32:25,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/cc01b7591c16424aa1adcaf8123f2b66, entries=200, sequenceid=62, filesize=38.6 K 2024-11-11T02:32:25,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/62c867129a734878aebe78446b657ef6 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/62c867129a734878aebe78446b657ef6 2024-11-11T02:32:25,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/62c867129a734878aebe78446b657ef6, entries=150, sequenceid=62, filesize=11.7 K 2024-11-11T02:32:25,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/1a7f8dc4bb2b433898a4d87a1e1a1255 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/1a7f8dc4bb2b433898a4d87a1e1a1255 2024-11-11T02:32:25,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/1a7f8dc4bb2b433898a4d87a1e1a1255, entries=150, sequenceid=62, filesize=11.7 K 2024-11-11T02:32:25,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 875c7d02961ba5720596e10e2c5e2c5e in 869ms, sequenceid=62, compaction requested=true 2024-11-11T02:32:25,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:25,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:32:25,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:25,327 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:25,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:32:25,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:25,327 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:25,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:32:25,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:25,328 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:25,328 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:25,328 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/A is initiating minor compaction (all files) 2024-11-11T02:32:25,328 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/B is initiating minor compaction (all files) 2024-11-11T02:32:25,328 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/B in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:25,328 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/A in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:25,328 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/5657c53483a84ecf83a065051cea565e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/8217b7328b284ed79a8debdcc0772995, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/62c867129a734878aebe78446b657ef6] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=35.2 K 2024-11-11T02:32:25,328 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/ec6101a136924812a6c3a037d88091fa, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/cb397d7505394396866b4700b3c4855b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/cc01b7591c16424aa1adcaf8123f2b66] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=107.5 K 2024-11-11T02:32:25,328 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:25,328 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/ec6101a136924812a6c3a037d88091fa, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/cb397d7505394396866b4700b3c4855b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/cc01b7591c16424aa1adcaf8123f2b66] 2024-11-11T02:32:25,328 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec6101a136924812a6c3a037d88091fa, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731292340981 2024-11-11T02:32:25,328 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 5657c53483a84ecf83a065051cea565e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731292340981 2024-11-11T02:32:25,329 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 8217b7328b284ed79a8debdcc0772995, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731292341065 2024-11-11T02:32:25,329 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb397d7505394396866b4700b3c4855b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731292341065 2024-11-11T02:32:25,329 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 62c867129a734878aebe78446b657ef6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731292342252 2024-11-11T02:32:25,329 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc01b7591c16424aa1adcaf8123f2b66, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731292342251 2024-11-11T02:32:25,334 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:25,337 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#B#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:25,338 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/0d064343443b46609fdb490a1b1dc77e is 50, key is test_row_0/B:col10/1731292342265/Put/seqid=0 2024-11-11T02:32:25,343 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411113673ba67343648b8830b96e3ff589159_875c7d02961ba5720596e10e2c5e2c5e store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:25,345 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411113673ba67343648b8830b96e3ff589159_875c7d02961ba5720596e10e2c5e2c5e, store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:25,346 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113673ba67343648b8830b96e3ff589159_875c7d02961ba5720596e10e2c5e2c5e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:25,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742147_1323 (size=12104) 2024-11-11T02:32:25,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742148_1324 (size=4469) 2024-11-11T02:32:25,354 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#A#compaction#270 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:25,355 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/ff471b47da6c4e38a357183b4dfa3fdd is 175, key is test_row_0/A:col10/1731292342265/Put/seqid=0 2024-11-11T02:32:25,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742149_1325 (size=31058) 2024-11-11T02:32:25,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T02:32:25,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:25,443 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T02:32:25,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:25,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:25,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:25,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:25,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:25,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:25,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110fc69b8396d04738b3db3f7c71a8d77b_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292344532/Put/seqid=0 2024-11-11T02:32:25,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742150_1326 (size=12154) 2024-11-11T02:32:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T02:32:25,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:25,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292405709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292405710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292405719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:25,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292405720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:25,752 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/0d064343443b46609fdb490a1b1dc77e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/0d064343443b46609fdb490a1b1dc77e 2024-11-11T02:32:25,756 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/B of 875c7d02961ba5720596e10e2c5e2c5e into 0d064343443b46609fdb490a1b1dc77e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:25,756 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:25,756 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/B, priority=13, startTime=1731292345327; duration=0sec 2024-11-11T02:32:25,756 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:25,756 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:B 2024-11-11T02:32:25,756 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:25,757 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:25,757 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/C is initiating minor compaction (all files) 2024-11-11T02:32:25,757 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/C in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:25,757 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/49f952f6b34f4267a770b23460c94069, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/b1cf6baa2d7641ff9090d652caa2073c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/1a7f8dc4bb2b433898a4d87a1e1a1255] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=35.2 K 2024-11-11T02:32:25,757 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 49f952f6b34f4267a770b23460c94069, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731292340981 2024-11-11T02:32:25,758 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting b1cf6baa2d7641ff9090d652caa2073c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731292341065 2024-11-11T02:32:25,758 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a7f8dc4bb2b433898a4d87a1e1a1255, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731292342252 2024-11-11T02:32:25,761 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/ff471b47da6c4e38a357183b4dfa3fdd as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/ff471b47da6c4e38a357183b4dfa3fdd 2024-11-11T02:32:25,765 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#C#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:25,765 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/A of 875c7d02961ba5720596e10e2c5e2c5e into ff471b47da6c4e38a357183b4dfa3fdd(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:25,765 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:25,765 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/A, priority=13, startTime=1731292345327; duration=0sec 2024-11-11T02:32:25,765 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:25,765 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:A 2024-11-11T02:32:25,765 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/1cf3d5aa891b4887a17373ffa6f7d7a8 is 50, key is test_row_0/C:col10/1731292342265/Put/seqid=0 2024-11-11T02:32:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T02:32:25,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742151_1327 (size=12104) 2024-11-11T02:32:25,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292405835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:25,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292405835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:25,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292405835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:25,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292405836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:25,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:25,856 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110fc69b8396d04738b3db3f7c71a8d77b_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110fc69b8396d04738b3db3f7c71a8d77b_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/4ccecb5f402b4d2da731a7d274da70c5, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/4ccecb5f402b4d2da731a7d274da70c5 is 175, key is test_row_0/A:col10/1731292344532/Put/seqid=0 2024-11-11T02:32:25,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742152_1328 (size=30955) 2024-11-11T02:32:26,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292406045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:26,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292406045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:26,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292406045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:26,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292406056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:26,172 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/1cf3d5aa891b4887a17373ffa6f7d7a8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/1cf3d5aa891b4887a17373ffa6f7d7a8 2024-11-11T02:32:26,176 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/C of 875c7d02961ba5720596e10e2c5e2c5e into 1cf3d5aa891b4887a17373ffa6f7d7a8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:26,176 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:26,176 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/C, priority=13, startTime=1731292345327; duration=0sec 2024-11-11T02:32:26,176 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:26,176 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:C 2024-11-11T02:32:26,261 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/4ccecb5f402b4d2da731a7d274da70c5 2024-11-11T02:32:26,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/7408731199b34b2ba4ac21c2a8dff69a is 50, key is test_row_0/B:col10/1731292344532/Put/seqid=0 2024-11-11T02:32:26,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742153_1329 (size=12001) 2024-11-11T02:32:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T02:32:26,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292406365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:26,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292406365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:26,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292406365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:26,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292406376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:26,671 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/7408731199b34b2ba4ac21c2a8dff69a 2024-11-11T02:32:26,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/76088e198cde4188be9349daaf8dd500 is 50, key is test_row_0/C:col10/1731292344532/Put/seqid=0 2024-11-11T02:32:26,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742154_1330 (size=12001) 2024-11-11T02:32:26,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292406875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:26,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292406875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:26,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292406885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:26,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292406896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:27,082 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/76088e198cde4188be9349daaf8dd500 2024-11-11T02:32:27,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/4ccecb5f402b4d2da731a7d274da70c5 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/4ccecb5f402b4d2da731a7d274da70c5 2024-11-11T02:32:27,089 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/4ccecb5f402b4d2da731a7d274da70c5, entries=150, sequenceid=82, filesize=30.2 K 2024-11-11T02:32:27,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/7408731199b34b2ba4ac21c2a8dff69a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/7408731199b34b2ba4ac21c2a8dff69a 2024-11-11T02:32:27,093 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/7408731199b34b2ba4ac21c2a8dff69a, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T02:32:27,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/76088e198cde4188be9349daaf8dd500 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/76088e198cde4188be9349daaf8dd500 2024-11-11T02:32:27,097 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/76088e198cde4188be9349daaf8dd500, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T02:32:27,097 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 875c7d02961ba5720596e10e2c5e2c5e in 1654ms, sequenceid=82, compaction requested=false 2024-11-11T02:32:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-11T02:32:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-11T02:32:27,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-11T02:32:27,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9610 sec 2024-11-11T02:32:27,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.9650 sec 2024-11-11T02:32:27,210 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T02:32:27,210 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T02:32:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T02:32:27,296 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:27,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-11T02:32:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T02:32:27,297 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:27,298 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:27,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T02:32:27,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T02:32:27,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:27,450 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T02:32:27,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:27,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:27,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:27,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:27,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:27,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:27,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114a0d927ce9cb48198f1f8eeee99ba9f4_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292345705/Put/seqid=0 2024-11-11T02:32:27,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742155_1331 (size=12154) 2024-11-11T02:32:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T02:32:27,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:27,865 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114a0d927ce9cb48198f1f8eeee99ba9f4_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114a0d927ce9cb48198f1f8eeee99ba9f4_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:27,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/e2ab3317c9bd484c881cf9e854191919, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:27,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/e2ab3317c9bd484c881cf9e854191919 is 175, key is test_row_0/A:col10/1731292345705/Put/seqid=0 2024-11-11T02:32:27,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742156_1332 (size=30955) 2024-11-11T02:32:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:27,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T02:32:27,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292407937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:27,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292407937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:27,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292407939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:27,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292407940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:27,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:27,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:27,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:28,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292408055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:28,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292408056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292408056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:28,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292408056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:28,270 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/e2ab3317c9bd484c881cf9e854191919 2024-11-11T02:32:28,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292408265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:28,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292408265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292408266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:28,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292408266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:28,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/0a78aa756c324ef0a9573e292c646c45 is 50, key is test_row_0/B:col10/1731292345705/Put/seqid=0 2024-11-11T02:32:28,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742157_1333 (size=12001) 2024-11-11T02:32:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T02:32:28,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292408575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:28,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:28,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292408586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:28,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292408586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292408586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:28,678 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/0a78aa756c324ef0a9573e292c646c45 2024-11-11T02:32:28,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/697188eddcfb4407a3a8dc4c754f94d1 is 50, key is test_row_0/C:col10/1731292345705/Put/seqid=0 2024-11-11T02:32:28,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742158_1334 (size=12001) 2024-11-11T02:32:28,688 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/697188eddcfb4407a3a8dc4c754f94d1 2024-11-11T02:32:28,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/e2ab3317c9bd484c881cf9e854191919 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/e2ab3317c9bd484c881cf9e854191919 2024-11-11T02:32:28,696 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/e2ab3317c9bd484c881cf9e854191919, entries=150, sequenceid=97, filesize=30.2 K 2024-11-11T02:32:28,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/0a78aa756c324ef0a9573e292c646c45 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/0a78aa756c324ef0a9573e292c646c45 2024-11-11T02:32:28,701 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/0a78aa756c324ef0a9573e292c646c45, entries=150, sequenceid=97, filesize=11.7 K 2024-11-11T02:32:28,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/697188eddcfb4407a3a8dc4c754f94d1 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/697188eddcfb4407a3a8dc4c754f94d1 2024-11-11T02:32:28,704 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/697188eddcfb4407a3a8dc4c754f94d1, entries=150, sequenceid=97, filesize=11.7 K 2024-11-11T02:32:28,705 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 875c7d02961ba5720596e10e2c5e2c5e in 1255ms, sequenceid=97, compaction requested=true 2024-11-11T02:32:28,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:28,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:28,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-11T02:32:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-11T02:32:28,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-11T02:32:28,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-11T02:32:28,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4130 sec 2024-11-11T02:32:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:29,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T02:32:29,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:29,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:29,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:29,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:29,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:29,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:29,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111170a203d90ef34d14b318f2d127de542c_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292349090/Put/seqid=0 2024-11-11T02:32:29,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742159_1335 (size=14594) 2024-11-11T02:32:29,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292409117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292409118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292409119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292409130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292409235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292409235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292409235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292409245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731292409325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,332 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in 875c7d02961ba5720596e10e2c5e2c5e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:32:29,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T02:32:29,445 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:29,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-11T02:32:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T02:32:29,447 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:29,448 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:29,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:29,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292409445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292409445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292409445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292409456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,504 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:29,507 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111170a203d90ef34d14b318f2d127de542c_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111170a203d90ef34d14b318f2d127de542c_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:29,508 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/6e5751b919be4a68a4163701d30b64c9, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:29,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/6e5751b919be4a68a4163701d30b64c9 is 175, key is test_row_0/A:col10/1731292349090/Put/seqid=0 2024-11-11T02:32:29,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742160_1336 (size=39549) 2024-11-11T02:32:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T02:32:29,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T02:32:29,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:29,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:29,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:29,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:29,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:29,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T02:32:29,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:29,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:29,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:29,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:29,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T02:32:29,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292409755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292409755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292409765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292409776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:29,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T02:32:29,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:29,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:29,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:29,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:29,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:29,912 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/6e5751b919be4a68a4163701d30b64c9 2024-11-11T02:32:29,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/bf801db1df424c0ea7874ce0ec9164b8 is 50, key is test_row_0/B:col10/1731292349090/Put/seqid=0 2024-11-11T02:32:29,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742161_1337 (size=12001) 2024-11-11T02:32:30,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T02:32:30,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:30,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:30,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:30,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T02:32:30,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T02:32:30,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:30,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:30,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:30,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292410275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:30,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292410275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:30,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292410285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:30,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292410296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:30,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:30,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:30,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:30,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/bf801db1df424c0ea7874ce0ec9164b8 2024-11-11T02:32:30,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/804192b99a6b40b5ab232e09f6c3c432 is 50, key is test_row_0/C:col10/1731292349090/Put/seqid=0 2024-11-11T02:32:30,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742162_1338 (size=12001) 2024-11-11T02:32:30,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T02:32:30,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:30,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:30,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:30,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T02:32:30,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:30,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:30,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:30,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T02:32:30,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T02:32:30,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:30,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:30,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:30,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:30,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/804192b99a6b40b5ab232e09f6c3c432 2024-11-11T02:32:30,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/6e5751b919be4a68a4163701d30b64c9 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/6e5751b919be4a68a4163701d30b64c9 2024-11-11T02:32:30,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/6e5751b919be4a68a4163701d30b64c9, entries=200, sequenceid=118, filesize=38.6 K 2024-11-11T02:32:30,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/bf801db1df424c0ea7874ce0ec9164b8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/bf801db1df424c0ea7874ce0ec9164b8 2024-11-11T02:32:30,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/bf801db1df424c0ea7874ce0ec9164b8, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T02:32:30,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/804192b99a6b40b5ab232e09f6c3c432 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/804192b99a6b40b5ab232e09f6c3c432 2024-11-11T02:32:30,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/804192b99a6b40b5ab232e09f6c3c432, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T02:32:30,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 875c7d02961ba5720596e10e2c5e2c5e in 1656ms, sequenceid=118, compaction requested=true 2024-11-11T02:32:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:32:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:32:30,748 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:32:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:30,748 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:32:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:32:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:30,749 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:32:30,749 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:32:30,749 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/B is initiating minor compaction (all files) 2024-11-11T02:32:30,749 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/A is initiating minor compaction (all files) 2024-11-11T02:32:30,749 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/B in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:30,749 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/A in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:30,749 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/0d064343443b46609fdb490a1b1dc77e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/7408731199b34b2ba4ac21c2a8dff69a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/0a78aa756c324ef0a9573e292c646c45, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/bf801db1df424c0ea7874ce0ec9164b8] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=47.0 K 2024-11-11T02:32:30,749 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/ff471b47da6c4e38a357183b4dfa3fdd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/4ccecb5f402b4d2da731a7d274da70c5, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/e2ab3317c9bd484c881cf9e854191919, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/6e5751b919be4a68a4163701d30b64c9] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=129.4 K 2024-11-11T02:32:30,749 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:30,749 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/ff471b47da6c4e38a357183b4dfa3fdd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/4ccecb5f402b4d2da731a7d274da70c5, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/e2ab3317c9bd484c881cf9e854191919, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/6e5751b919be4a68a4163701d30b64c9] 2024-11-11T02:32:30,749 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d064343443b46609fdb490a1b1dc77e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731292342252 2024-11-11T02:32:30,749 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff471b47da6c4e38a357183b4dfa3fdd, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731292342252 2024-11-11T02:32:30,749 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 7408731199b34b2ba4ac21c2a8dff69a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731292344529 2024-11-11T02:32:30,749 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ccecb5f402b4d2da731a7d274da70c5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731292344529 2024-11-11T02:32:30,749 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a78aa756c324ef0a9573e292c646c45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731292345704 2024-11-11T02:32:30,750 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2ab3317c9bd484c881cf9e854191919, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731292345704 2024-11-11T02:32:30,750 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting bf801db1df424c0ea7874ce0ec9164b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292347935 2024-11-11T02:32:30,750 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e5751b919be4a68a4163701d30b64c9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292347935 2024-11-11T02:32:30,756 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:30,757 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#B#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:30,758 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/e7102085bf7a4968933e5a95477b3524 is 50, key is test_row_0/B:col10/1731292349090/Put/seqid=0 2024-11-11T02:32:30,760 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411112f81b72794d343768f3c940c4acf0681_875c7d02961ba5720596e10e2c5e2c5e store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:30,763 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411112f81b72794d343768f3c940c4acf0681_875c7d02961ba5720596e10e2c5e2c5e, store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:30,763 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112f81b72794d343768f3c940c4acf0681_875c7d02961ba5720596e10e2c5e2c5e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:30,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742164_1340 (size=4469) 2024-11-11T02:32:30,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742163_1339 (size=12241) 2024-11-11T02:32:30,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T02:32:30,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:30,822 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T02:32:30,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:30,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:30,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:30,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:30,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:30,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:30,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111187f3c322e3f141988f692273f62ecfea_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292349129/Put/seqid=0 2024-11-11T02:32:30,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742165_1341 (size=12154) 2024-11-11T02:32:31,168 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#A#compaction#283 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:31,169 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/1802899eb7ff44c186ea06ef3de569ea is 175, key is test_row_0/A:col10/1731292349090/Put/seqid=0 2024-11-11T02:32:31,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742166_1342 (size=31195) 2024-11-11T02:32:31,173 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/e7102085bf7a4968933e5a95477b3524 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/e7102085bf7a4968933e5a95477b3524 2024-11-11T02:32:31,178 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/B of 875c7d02961ba5720596e10e2c5e2c5e into e7102085bf7a4968933e5a95477b3524(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:31,178 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:31,178 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/B, priority=12, startTime=1731292350748; duration=0sec 2024-11-11T02:32:31,178 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:31,178 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:B 2024-11-11T02:32:31,178 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:32:31,179 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:32:31,180 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/C is initiating minor compaction (all files) 2024-11-11T02:32:31,180 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/C in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:31,180 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/1cf3d5aa891b4887a17373ffa6f7d7a8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/76088e198cde4188be9349daaf8dd500, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/697188eddcfb4407a3a8dc4c754f94d1, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/804192b99a6b40b5ab232e09f6c3c432] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=47.0 K 2024-11-11T02:32:31,180 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cf3d5aa891b4887a17373ffa6f7d7a8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731292342252 2024-11-11T02:32:31,180 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 76088e198cde4188be9349daaf8dd500, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731292344529 2024-11-11T02:32:31,180 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 697188eddcfb4407a3a8dc4c754f94d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731292345704 2024-11-11T02:32:31,181 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 804192b99a6b40b5ab232e09f6c3c432, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292347935 2024-11-11T02:32:31,188 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:31,189 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/14b2a7f6251a45088afc2724a2330e3b is 50, key is test_row_0/C:col10/1731292349090/Put/seqid=0 2024-11-11T02:32:31,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742167_1343 (size=12241) 2024-11-11T02:32:31,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:31,237 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111187f3c322e3f141988f692273f62ecfea_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111187f3c322e3f141988f692273f62ecfea_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:31,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/35abbde42c384735aec9d5cc04cb552b, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:31,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/35abbde42c384735aec9d5cc04cb552b is 175, key is test_row_0/A:col10/1731292349129/Put/seqid=0 2024-11-11T02:32:31,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742168_1344 (size=30955) 2024-11-11T02:32:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:31,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:31,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292351329, max timestamp=1731292351382Num uniques 859; 2024-11-11T02:32:31,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:32:31,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292351329, max timestamp=1731292351382Num uniques 859; 2024-11-11T02:32:31,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:32:31,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292351329, max timestamp=1731292351382Num uniques 859; 2024-11-11T02:32:31,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:32:31,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292411421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292411422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:31,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292411435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:31,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:31,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292411436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:31,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:31,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292411545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:31,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:31,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292411545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:31,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:31,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292411555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:31,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:31,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292411556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:31,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:31,576 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/1802899eb7ff44c186ea06ef3de569ea as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/1802899eb7ff44c186ea06ef3de569ea 2024-11-11T02:32:31,580 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/A of 875c7d02961ba5720596e10e2c5e2c5e into 1802899eb7ff44c186ea06ef3de569ea(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:31,580 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:31,580 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/A, priority=12, startTime=1731292350748; duration=0sec 2024-11-11T02:32:31,580 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:31,580 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:A 2024-11-11T02:32:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T02:32:31,603 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/14b2a7f6251a45088afc2724a2330e3b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/14b2a7f6251a45088afc2724a2330e3b 2024-11-11T02:32:31,607 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/C of 875c7d02961ba5720596e10e2c5e2c5e into 14b2a7f6251a45088afc2724a2330e3b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:31,607 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:31,607 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/C, priority=12, startTime=1731292350748; duration=0sec 2024-11-11T02:32:31,607 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:31,607 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:C 2024-11-11T02:32:31,644 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/35abbde42c384735aec9d5cc04cb552b 2024-11-11T02:32:31,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/7f72f277114340cebb0aef688f4516e8 is 50, key is test_row_0/B:col10/1731292349129/Put/seqid=0 2024-11-11T02:32:31,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742169_1345 (size=12001) 2024-11-11T02:32:31,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292411765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:31,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292411765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:31,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292411765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:31,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292411766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,054 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/7f72f277114340cebb0aef688f4516e8 2024-11-11T02:32:32,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/55dcd18aaa764581827bead7af18312f is 50, key is test_row_0/C:col10/1731292349129/Put/seqid=0 2024-11-11T02:32:32,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742170_1346 (size=12001) 2024-11-11T02:32:32,068 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/55dcd18aaa764581827bead7af18312f 2024-11-11T02:32:32,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/35abbde42c384735aec9d5cc04cb552b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/35abbde42c384735aec9d5cc04cb552b 2024-11-11T02:32:32,075 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/35abbde42c384735aec9d5cc04cb552b, entries=150, sequenceid=129, filesize=30.2 K 2024-11-11T02:32:32,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/7f72f277114340cebb0aef688f4516e8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/7f72f277114340cebb0aef688f4516e8 2024-11-11T02:32:32,079 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/7f72f277114340cebb0aef688f4516e8, entries=150, sequenceid=129, filesize=11.7 K 2024-11-11T02:32:32,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/55dcd18aaa764581827bead7af18312f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/55dcd18aaa764581827bead7af18312f 2024-11-11T02:32:32,086 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/55dcd18aaa764581827bead7af18312f, entries=150, sequenceid=129, filesize=11.7 K 2024-11-11T02:32:32,087 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 875c7d02961ba5720596e10e2c5e2c5e in 1265ms, sequenceid=129, compaction requested=false 2024-11-11T02:32:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-11T02:32:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-11T02:32:32,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-11T02:32:32,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6400 sec 2024-11-11T02:32:32,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.6430 sec 2024-11-11T02:32:32,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:32,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T02:32:32,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:32,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:32,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:32,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:32,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:32,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:32,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111140aa1b3dfaf40308e2a6618612634fa_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292351433/Put/seqid=0 2024-11-11T02:32:32,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742171_1347 (size=14794) 2024-11-11T02:32:32,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292412104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292412110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292412110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292412111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292412215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292412226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292412226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292412226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292412436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292412445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292412445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292412445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,506 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:32,509 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111140aa1b3dfaf40308e2a6618612634fa_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111140aa1b3dfaf40308e2a6618612634fa_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:32,510 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/f196030c82c64961861aa57e4a505cee, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:32,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/f196030c82c64961861aa57e4a505cee is 175, key is test_row_0/A:col10/1731292351433/Put/seqid=0 2024-11-11T02:32:32,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742172_1348 (size=39749) 2024-11-11T02:32:32,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292412756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292412766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292412766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292412766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:32,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:32,914 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/f196030c82c64961861aa57e4a505cee 2024-11-11T02:32:32,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/30f7d0bccf834d7dbfa1ff0ef188091a is 50, key is test_row_0/B:col10/1731292351433/Put/seqid=0 2024-11-11T02:32:32,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742173_1349 (size=12151) 2024-11-11T02:32:33,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:33,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292413276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:33,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:33,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292413276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:33,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:33,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292413286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:33,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:33,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292413286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:33,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/30f7d0bccf834d7dbfa1ff0ef188091a 2024-11-11T02:32:33,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/df2f33c054c54620a58b8234cf7ea797 is 50, key is test_row_0/C:col10/1731292351433/Put/seqid=0 2024-11-11T02:32:33,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742174_1350 (size=12151) 2024-11-11T02:32:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T02:32:33,606 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:33,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-11T02:32:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T02:32:33,608 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:33,608 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:33,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T02:32:33,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/df2f33c054c54620a58b8234cf7ea797 2024-11-11T02:32:33,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/f196030c82c64961861aa57e4a505cee as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f196030c82c64961861aa57e4a505cee 2024-11-11T02:32:33,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f196030c82c64961861aa57e4a505cee, entries=200, sequenceid=169, filesize=38.8 K 2024-11-11T02:32:33,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/30f7d0bccf834d7dbfa1ff0ef188091a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/30f7d0bccf834d7dbfa1ff0ef188091a 2024-11-11T02:32:33,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/30f7d0bccf834d7dbfa1ff0ef188091a, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T02:32:33,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/df2f33c054c54620a58b8234cf7ea797 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/df2f33c054c54620a58b8234cf7ea797 2024-11-11T02:32:33,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/df2f33c054c54620a58b8234cf7ea797, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T02:32:33,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 875c7d02961ba5720596e10e2c5e2c5e in 1652ms, sequenceid=169, compaction requested=true 2024-11-11T02:32:33,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:33,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:32:33,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:33,748 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:33,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:32:33,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:33,748 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:33,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:32:33,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:33,748 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:33,748 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:33,748 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/B is initiating minor compaction (all files) 2024-11-11T02:32:33,748 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/A is initiating minor compaction (all files) 2024-11-11T02:32:33,748 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/B in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:33,748 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/A in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:33,748 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/e7102085bf7a4968933e5a95477b3524, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/7f72f277114340cebb0aef688f4516e8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/30f7d0bccf834d7dbfa1ff0ef188091a] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=35.5 K 2024-11-11T02:32:33,748 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/1802899eb7ff44c186ea06ef3de569ea, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/35abbde42c384735aec9d5cc04cb552b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f196030c82c64961861aa57e4a505cee] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=99.5 K 2024-11-11T02:32:33,749 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:33,749 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/1802899eb7ff44c186ea06ef3de569ea, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/35abbde42c384735aec9d5cc04cb552b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f196030c82c64961861aa57e4a505cee] 2024-11-11T02:32:33,749 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting e7102085bf7a4968933e5a95477b3524, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292347935 2024-11-11T02:32:33,749 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1802899eb7ff44c186ea06ef3de569ea, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292347935 2024-11-11T02:32:33,749 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f72f277114340cebb0aef688f4516e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731292349115 2024-11-11T02:32:33,749 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35abbde42c384735aec9d5cc04cb552b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731292349115 2024-11-11T02:32:33,749 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 30f7d0bccf834d7dbfa1ff0ef188091a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731292351433 2024-11-11T02:32:33,749 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting f196030c82c64961861aa57e4a505cee, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731292351420 2024-11-11T02:32:33,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T02:32:33,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:33,761 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:33,761 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:32:33,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:33,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:33,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:33,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:33,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:33,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:33,762 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#B#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:33,762 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/71f1f5cc158d41fa8b60ed0195da0a4f is 50, key is test_row_0/B:col10/1731292351433/Put/seqid=0 2024-11-11T02:32:33,764 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411118adc5b1a25064839afea16caff8cf9de_875c7d02961ba5720596e10e2c5e2c5e store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:33,766 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411118adc5b1a25064839afea16caff8cf9de_875c7d02961ba5720596e10e2c5e2c5e, store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:33,766 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118adc5b1a25064839afea16caff8cf9de_875c7d02961ba5720596e10e2c5e2c5e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c5cb647e404e4c4d92e367a8c2dd9e30_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292352095/Put/seqid=0 2024-11-11T02:32:33,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742175_1351 (size=12493) 2024-11-11T02:32:33,778 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/71f1f5cc158d41fa8b60ed0195da0a4f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/71f1f5cc158d41fa8b60ed0195da0a4f 2024-11-11T02:32:33,783 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/B of 875c7d02961ba5720596e10e2c5e2c5e into 71f1f5cc158d41fa8b60ed0195da0a4f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:33,784 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:33,784 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/B, priority=13, startTime=1731292353748; duration=0sec 2024-11-11T02:32:33,784 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:33,784 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:B 2024-11-11T02:32:33,784 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:33,785 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:33,785 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/C is initiating minor compaction (all files) 2024-11-11T02:32:33,785 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/C in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:33,785 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/14b2a7f6251a45088afc2724a2330e3b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/55dcd18aaa764581827bead7af18312f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/df2f33c054c54620a58b8234cf7ea797] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=35.5 K 2024-11-11T02:32:33,785 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 14b2a7f6251a45088afc2724a2330e3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731292347935 2024-11-11T02:32:33,785 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 55dcd18aaa764581827bead7af18312f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731292349115 2024-11-11T02:32:33,786 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting df2f33c054c54620a58b8234cf7ea797, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731292351433 2024-11-11T02:32:33,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742176_1352 (size=4469) 2024-11-11T02:32:33,787 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#A#compaction#292 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:33,787 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/f73e0b0403b94999b9b9fcea4e696339 is 175, key is test_row_0/A:col10/1731292351433/Put/seqid=0 2024-11-11T02:32:33,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742177_1353 (size=12304) 2024-11-11T02:32:33,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:33,793 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c5cb647e404e4c4d92e367a8c2dd9e30_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c5cb647e404e4c4d92e367a8c2dd9e30_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:33,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742178_1354 (size=31447) 2024-11-11T02:32:33,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/0200e4d30ed2452c8ba7332838f6f2eb, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:33,795 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#C#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:33,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/0200e4d30ed2452c8ba7332838f6f2eb is 175, key is test_row_0/A:col10/1731292352095/Put/seqid=0 2024-11-11T02:32:33,795 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/51ca194f218343f5ac819ddd2d628432 is 50, key is test_row_0/C:col10/1731292351433/Put/seqid=0 2024-11-11T02:32:33,799 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/f73e0b0403b94999b9b9fcea4e696339 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f73e0b0403b94999b9b9fcea4e696339 2024-11-11T02:32:33,806 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/A of 875c7d02961ba5720596e10e2c5e2c5e into f73e0b0403b94999b9b9fcea4e696339(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:33,806 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:33,806 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/A, priority=13, startTime=1731292353747; duration=0sec 2024-11-11T02:32:33,806 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:33,806 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:A 2024-11-11T02:32:33,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742179_1355 (size=31105) 2024-11-11T02:32:33,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742180_1356 (size=12493) 2024-11-11T02:32:33,813 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/51ca194f218343f5ac819ddd2d628432 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/51ca194f218343f5ac819ddd2d628432 2024-11-11T02:32:33,818 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/C of 875c7d02961ba5720596e10e2c5e2c5e into 51ca194f218343f5ac819ddd2d628432(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:33,818 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:33,818 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/C, priority=13, startTime=1731292353748; duration=0sec 2024-11-11T02:32:33,818 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:33,818 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:C 2024-11-11T02:32:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T02:32:34,209 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/0200e4d30ed2452c8ba7332838f6f2eb 2024-11-11T02:32:34,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/04deeb84bdbc4f569b3544ed1113085a is 50, key is test_row_0/B:col10/1731292352095/Put/seqid=0 2024-11-11T02:32:34,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742181_1357 (size=12151) 2024-11-11T02:32:34,224 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/04deeb84bdbc4f569b3544ed1113085a 2024-11-11T02:32:34,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T02:32:34,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/a124b6a3d9184650ab91e8589e5414d9 is 50, key is test_row_0/C:col10/1731292352095/Put/seqid=0 2024-11-11T02:32:34,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742182_1358 (size=12151) 2024-11-11T02:32:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:34,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:34,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292354327, max timestamp=1731292354340Num uniques 859; 2024-11-11T02:32:34,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292354327, max timestamp=1731292354340Num uniques 859; 2024-11-11T02:32:34,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292354327, max timestamp=1731292354340Num uniques 859; 2024-11-11T02:32:34,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:34,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:34,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:34,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292414377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292414377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292414383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292414383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292414485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292414496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292414506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292414506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,690 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/a124b6a3d9184650ab91e8589e5414d9 2024-11-11T02:32:34,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/0200e4d30ed2452c8ba7332838f6f2eb as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/0200e4d30ed2452c8ba7332838f6f2eb 2024-11-11T02:32:34,697 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/0200e4d30ed2452c8ba7332838f6f2eb, entries=150, sequenceid=179, filesize=30.4 K 2024-11-11T02:32:34,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/04deeb84bdbc4f569b3544ed1113085a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/04deeb84bdbc4f569b3544ed1113085a 2024-11-11T02:32:34,700 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/04deeb84bdbc4f569b3544ed1113085a, entries=150, sequenceid=179, filesize=11.9 K 2024-11-11T02:32:34,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/a124b6a3d9184650ab91e8589e5414d9 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/a124b6a3d9184650ab91e8589e5414d9 2024-11-11T02:32:34,704 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/a124b6a3d9184650ab91e8589e5414d9, entries=150, sequenceid=179, filesize=11.9 K 2024-11-11T02:32:34,705 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 875c7d02961ba5720596e10e2c5e2c5e in 945ms, sequenceid=179, compaction requested=false 2024-11-11T02:32:34,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:34,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:34,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-11T02:32:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-11T02:32:34,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-11T02:32:34,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0980 sec 2024-11-11T02:32:34,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.1020 sec 2024-11-11T02:32:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:34,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-11T02:32:34,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:34,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:34,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:34,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:34,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:34,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:34,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113b9b38d891e14020a4594306051386b5_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292354368/Put/seqid=0 2024-11-11T02:32:34,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742183_1359 (size=14794) 2024-11-11T02:32:34,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292414735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T02:32:34,746 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:34,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-11T02:32:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T02:32:34,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292414737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,748 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:34,749 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:34,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:34,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292414738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292414739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T02:32:34,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292414855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292414856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292414856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292414856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:34,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T02:32:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:34,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T02:32:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:35,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T02:32:35,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292415065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:35,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:35,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292415076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292415076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292415076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:35,136 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:35,139 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113b9b38d891e14020a4594306051386b5_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113b9b38d891e14020a4594306051386b5_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:35,140 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/d4205093e4b94d40906fb57b1ddb3ca4, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:35,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/d4205093e4b94d40906fb57b1ddb3ca4 is 175, key is test_row_0/A:col10/1731292354368/Put/seqid=0 2024-11-11T02:32:35,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742184_1360 (size=39749) 2024-11-11T02:32:35,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T02:32:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:35,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T02:32:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:35,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T02:32:35,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292415385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:35,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292415396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:35,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292415396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292415396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:35,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T02:32:35,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:35,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:35,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:35,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,543 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/d4205093e4b94d40906fb57b1ddb3ca4 2024-11-11T02:32:35,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/ec27cfc7e6cd44259f269dfc22d540e9 is 50, key is test_row_0/B:col10/1731292354368/Put/seqid=0 2024-11-11T02:32:35,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742185_1361 (size=12151) 2024-11-11T02:32:35,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T02:32:35,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:35,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:35,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:35,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T02:32:35,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:35,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:35,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:35,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T02:32:35,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292415905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:35,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:35,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292415916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:35,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292415916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292415916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:35,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/ec27cfc7e6cd44259f269dfc22d540e9 2024-11-11T02:32:35,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/c276ad5891b34651843ca643940f503a is 50, key is test_row_0/C:col10/1731292354368/Put/seqid=0 2024-11-11T02:32:35,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742186_1362 (size=12151) 2024-11-11T02:32:35,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T02:32:35,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:35,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:35,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:35,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:36,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T02:32:36,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:36,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:36,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:36,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:36,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:36,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T02:32:36,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:36,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:36,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:36,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:36,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:36,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/c276ad5891b34651843ca643940f503a 2024-11-11T02:32:36,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/d4205093e4b94d40906fb57b1ddb3ca4 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/d4205093e4b94d40906fb57b1ddb3ca4 2024-11-11T02:32:36,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/d4205093e4b94d40906fb57b1ddb3ca4, entries=200, sequenceid=219, filesize=38.8 K 2024-11-11T02:32:36,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/ec27cfc7e6cd44259f269dfc22d540e9 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/ec27cfc7e6cd44259f269dfc22d540e9 2024-11-11T02:32:36,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/ec27cfc7e6cd44259f269dfc22d540e9, entries=150, sequenceid=219, filesize=11.9 K 2024-11-11T02:32:36,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/c276ad5891b34651843ca643940f503a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c276ad5891b34651843ca643940f503a 2024-11-11T02:32:36,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c276ad5891b34651843ca643940f503a, entries=150, sequenceid=219, filesize=11.9 K 2024-11-11T02:32:36,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 875c7d02961ba5720596e10e2c5e2c5e in 1664ms, sequenceid=219, compaction requested=true 2024-11-11T02:32:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:32:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:36,379 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:32:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:36,379 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:32:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:36,380 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:36,380 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/B is initiating minor compaction (all files) 2024-11-11T02:32:36,380 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/B in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:36,380 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/71f1f5cc158d41fa8b60ed0195da0a4f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/04deeb84bdbc4f569b3544ed1113085a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/ec27cfc7e6cd44259f269dfc22d540e9] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=35.9 K 2024-11-11T02:32:36,380 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:36,380 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/A is initiating minor compaction (all files) 2024-11-11T02:32:36,380 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/A in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:36,380 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f73e0b0403b94999b9b9fcea4e696339, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/0200e4d30ed2452c8ba7332838f6f2eb, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/d4205093e4b94d40906fb57b1ddb3ca4] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=99.9 K 2024-11-11T02:32:36,380 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:36,380 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f73e0b0403b94999b9b9fcea4e696339, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/0200e4d30ed2452c8ba7332838f6f2eb, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/d4205093e4b94d40906fb57b1ddb3ca4] 2024-11-11T02:32:36,381 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 71f1f5cc158d41fa8b60ed0195da0a4f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731292351433 2024-11-11T02:32:36,381 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting f73e0b0403b94999b9b9fcea4e696339, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731292351433 2024-11-11T02:32:36,381 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 04deeb84bdbc4f569b3544ed1113085a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731292352095 2024-11-11T02:32:36,381 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0200e4d30ed2452c8ba7332838f6f2eb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731292352095 2024-11-11T02:32:36,381 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting ec27cfc7e6cd44259f269dfc22d540e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731292354368 2024-11-11T02:32:36,382 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4205093e4b94d40906fb57b1ddb3ca4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731292354368 2024-11-11T02:32:36,386 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:36,389 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#B#compaction#301 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:36,389 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/047e0613a1914747ac485f8d0681a6c2 is 50, key is test_row_0/B:col10/1731292354368/Put/seqid=0 2024-11-11T02:32:36,390 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111d89443f45ba8487d920e7acc31702aa8_875c7d02961ba5720596e10e2c5e2c5e store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:36,393 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111d89443f45ba8487d920e7acc31702aa8_875c7d02961ba5720596e10e2c5e2c5e, store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:36,393 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d89443f45ba8487d920e7acc31702aa8_875c7d02961ba5720596e10e2c5e2c5e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:36,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742187_1363 (size=12595) 2024-11-11T02:32:36,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742188_1364 (size=4469) 2024-11-11T02:32:36,404 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#A#compaction#300 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:36,405 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/c15ffc18a9554b16b75b20c3f96d3bb5 is 175, key is test_row_0/A:col10/1731292354368/Put/seqid=0 2024-11-11T02:32:36,406 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/047e0613a1914747ac485f8d0681a6c2 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/047e0613a1914747ac485f8d0681a6c2 2024-11-11T02:32:36,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742189_1365 (size=31549) 2024-11-11T02:32:36,412 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/B of 875c7d02961ba5720596e10e2c5e2c5e into 047e0613a1914747ac485f8d0681a6c2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:36,412 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:36,412 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/B, priority=13, startTime=1731292356379; duration=0sec 2024-11-11T02:32:36,412 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:36,412 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:B 2024-11-11T02:32:36,412 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:36,412 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:36,412 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/C is initiating minor compaction (all files) 2024-11-11T02:32:36,412 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/C in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:36,413 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/51ca194f218343f5ac819ddd2d628432, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/a124b6a3d9184650ab91e8589e5414d9, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c276ad5891b34651843ca643940f503a] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=35.9 K 2024-11-11T02:32:36,413 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 51ca194f218343f5ac819ddd2d628432, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731292351433 2024-11-11T02:32:36,413 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting a124b6a3d9184650ab91e8589e5414d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731292352095 2024-11-11T02:32:36,413 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting c276ad5891b34651843ca643940f503a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731292354368 2024-11-11T02:32:36,421 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:36,422 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/5fffa3b669f24723a9c46893aa5fc21c is 50, key is test_row_0/C:col10/1731292354368/Put/seqid=0 2024-11-11T02:32:36,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742190_1366 (size=12595) 2024-11-11T02:32:36,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T02:32:36,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:36,427 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T02:32:36,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:36,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:36,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:36,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:36,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:36,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:36,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111146ae9ff764ad4d4f8ef5f13f85dad705_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292354718/Put/seqid=0 2024-11-11T02:32:36,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742191_1367 (size=12304) 2024-11-11T02:32:36,811 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/c15ffc18a9554b16b75b20c3f96d3bb5 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/c15ffc18a9554b16b75b20c3f96d3bb5 2024-11-11T02:32:36,815 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/A of 875c7d02961ba5720596e10e2c5e2c5e into c15ffc18a9554b16b75b20c3f96d3bb5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:36,815 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:36,815 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/A, priority=13, startTime=1731292356379; duration=0sec 2024-11-11T02:32:36,815 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:36,815 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:A 2024-11-11T02:32:36,829 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/5fffa3b669f24723a9c46893aa5fc21c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/5fffa3b669f24723a9c46893aa5fc21c 2024-11-11T02:32:36,833 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/C of 875c7d02961ba5720596e10e2c5e2c5e into 5fffa3b669f24723a9c46893aa5fc21c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:36,833 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:36,833 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/C, priority=13, startTime=1731292356379; duration=0sec 2024-11-11T02:32:36,833 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:36,833 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:C 2024-11-11T02:32:36,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:36,842 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111146ae9ff764ad4d4f8ef5f13f85dad705_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146ae9ff764ad4d4f8ef5f13f85dad705_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:36,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/6f6790d2868049d9be580b7a81fc3f2e, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:36,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/6f6790d2868049d9be580b7a81fc3f2e is 175, key is test_row_0/A:col10/1731292354718/Put/seqid=0 2024-11-11T02:32:36,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742192_1368 (size=31105) 2024-11-11T02:32:36,847 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/6f6790d2868049d9be580b7a81fc3f2e 2024-11-11T02:32:36,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/875838a981624ab9940390e886a0951f is 50, key is test_row_0/B:col10/1731292354718/Put/seqid=0 2024-11-11T02:32:36,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742193_1369 (size=12151) 2024-11-11T02:32:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T02:32:36,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:36,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:36,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292356954, max timestamp=1731292356972Num uniques 859; 2024-11-11T02:32:36,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292356954, max timestamp=1731292356972Num uniques 859; 2024-11-11T02:32:36,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292356954, max timestamp=1731292356972Num uniques 859; 2024-11-11T02:32:36,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:36,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:36,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:37,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292417005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292417006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292417014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292417016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292417126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292417126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292417126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292417135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,260 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/875838a981624ab9940390e886a0951f 2024-11-11T02:32:37,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/41221cc410d249e0a359bee9e0acc4df is 50, key is test_row_0/C:col10/1731292354718/Put/seqid=0 2024-11-11T02:32:37,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742194_1370 (size=12151) 2024-11-11T02:32:37,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292417346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292417346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292417346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292417355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292417656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292417656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292417656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,674 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/41221cc410d249e0a359bee9e0acc4df 2024-11-11T02:32:37,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292417665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:37,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/6f6790d2868049d9be580b7a81fc3f2e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/6f6790d2868049d9be580b7a81fc3f2e 2024-11-11T02:32:37,681 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/6f6790d2868049d9be580b7a81fc3f2e, entries=150, sequenceid=229, filesize=30.4 K 2024-11-11T02:32:37,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/875838a981624ab9940390e886a0951f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/875838a981624ab9940390e886a0951f 2024-11-11T02:32:37,684 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/875838a981624ab9940390e886a0951f, entries=150, sequenceid=229, filesize=11.9 K 2024-11-11T02:32:37,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/41221cc410d249e0a359bee9e0acc4df as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/41221cc410d249e0a359bee9e0acc4df 2024-11-11T02:32:37,688 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/41221cc410d249e0a359bee9e0acc4df, entries=150, sequenceid=229, filesize=11.9 K 2024-11-11T02:32:37,689 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 875c7d02961ba5720596e10e2c5e2c5e in 1261ms, sequenceid=229, compaction requested=false 2024-11-11T02:32:37,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:37,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:37,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-11T02:32:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-11T02:32:37,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-11T02:32:37,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9410 sec 2024-11-11T02:32:37,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.9450 sec 2024-11-11T02:32:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:38,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-11T02:32:38,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:38,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:32:38,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:38,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:32:38,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:38,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:32:38,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110cf1dd3ad2184791b4db3c88480383b6_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292357001/Put/seqid=0 2024-11-11T02:32:38,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742195_1371 (size=14994) 2024-11-11T02:32:38,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292418185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:38,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:38,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292418185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:38,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:38,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:38,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292418189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:38,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:38,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292418193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:38,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292418305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292418305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292418305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:38,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292418515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:38,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:38,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292418525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292418525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:38,582 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:38,585 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110cf1dd3ad2184791b4db3c88480383b6_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110cf1dd3ad2184791b4db3c88480383b6_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:38,586 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/76d1abb3d1d6453c90098437ebcc8272, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:38,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/76d1abb3d1d6453c90098437ebcc8272 is 175, key is test_row_0/A:col10/1731292357001/Put/seqid=0 2024-11-11T02:32:38,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742196_1372 (size=39949) 2024-11-11T02:32:38,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292418836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:38,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292418836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:38,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:38,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:38,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292418836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:38,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:38,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T02:32:38,916 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:38,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:38,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-11T02:32:38,918 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:38,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T02:32:38,918 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:38,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:38,989 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/76d1abb3d1d6453c90098437ebcc8272 2024-11-11T02:32:38,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/ee8d8ca9e61640dc86efe563fff8ab8d is 50, key is test_row_0/B:col10/1731292357001/Put/seqid=0 2024-11-11T02:32:38,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742197_1373 (size=12301) 2024-11-11T02:32:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T02:32:39,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T02:32:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:39,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292419206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:39,217 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in 875c7d02961ba5720596e10e2c5e2c5e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:32:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:39,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T02:32:39,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:39,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:39,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:39,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T02:32:39,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292419355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:39,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292419355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:39,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292419355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:39,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T02:32:39,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:39,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:39,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:39,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/ee8d8ca9e61640dc86efe563fff8ab8d 2024-11-11T02:32:39,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/c3a1dba2432546209521b854be653b4e is 50, key is test_row_0/C:col10/1731292357001/Put/seqid=0 2024-11-11T02:32:39,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742198_1374 (size=12301) 2024-11-11T02:32:39,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731292419415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:39,423 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in 875c7d02961ba5720596e10e2c5e2c5e of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18353 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:32:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:39,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T02:32:39,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:39,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:39,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:39,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T02:32:39,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T02:32:39,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:39,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:39,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:39,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:39,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/c3a1dba2432546209521b854be653b4e 2024-11-11T02:32:39,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/76d1abb3d1d6453c90098437ebcc8272 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/76d1abb3d1d6453c90098437ebcc8272 2024-11-11T02:32:39,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/76d1abb3d1d6453c90098437ebcc8272, entries=200, sequenceid=269, filesize=39.0 K 2024-11-11T02:32:39,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/ee8d8ca9e61640dc86efe563fff8ab8d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/ee8d8ca9e61640dc86efe563fff8ab8d 2024-11-11T02:32:39,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/ee8d8ca9e61640dc86efe563fff8ab8d, entries=150, sequenceid=269, filesize=12.0 K 2024-11-11T02:32:39,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/c3a1dba2432546209521b854be653b4e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c3a1dba2432546209521b854be653b4e 2024-11-11T02:32:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c3a1dba2432546209521b854be653b4e, entries=150, sequenceid=269, filesize=12.0 K 2024-11-11T02:32:39,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 875c7d02961ba5720596e10e2c5e2c5e in 1658ms, sequenceid=269, compaction requested=true 2024-11-11T02:32:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:32:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:32:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:39,830 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:32:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:39,830 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,831 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:39,831 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:39,831 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/B is initiating minor compaction (all files) 2024-11-11T02:32:39,831 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/A is initiating minor compaction (all files) 2024-11-11T02:32:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,831 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/B in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:39,831 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/A in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:39,831 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/c15ffc18a9554b16b75b20c3f96d3bb5, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/6f6790d2868049d9be580b7a81fc3f2e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/76d1abb3d1d6453c90098437ebcc8272] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=100.2 K 2024-11-11T02:32:39,831 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/047e0613a1914747ac485f8d0681a6c2, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/875838a981624ab9940390e886a0951f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/ee8d8ca9e61640dc86efe563fff8ab8d] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=36.2 K 2024-11-11T02:32:39,831 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:39,832 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/c15ffc18a9554b16b75b20c3f96d3bb5, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/6f6790d2868049d9be580b7a81fc3f2e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/76d1abb3d1d6453c90098437ebcc8272] 2024-11-11T02:32:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,832 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 047e0613a1914747ac485f8d0681a6c2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731292354368 2024-11-11T02:32:39,832 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting c15ffc18a9554b16b75b20c3f96d3bb5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731292354368 2024-11-11T02:32:39,832 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 875838a981624ab9940390e886a0951f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731292354718 2024-11-11T02:32:39,832 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f6790d2868049d9be580b7a81fc3f2e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731292354718 2024-11-11T02:32:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,832 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting ee8d8ca9e61640dc86efe563fff8ab8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731292357001 2024-11-11T02:32:39,832 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76d1abb3d1d6453c90098437ebcc8272, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731292357001 2024-11-11T02:32:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T02:32:39,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:39,834 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T02:32:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,840 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,840 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/8fb249b5707e4dd19be3090f1d03bb62 is 50, key is test_row_0/B:col10/1731292357001/Put/seqid=0 2024-11-11T02:32:39,841 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114565a98f5f9643aba22cdc8505795f28_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292358192/Put/seqid=0 2024-11-11T02:32:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,858 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411111462609fa58a4bc9a040ccf66057307b_875c7d02961ba5720596e10e2c5e2c5e store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,859 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411111462609fa58a4bc9a040ccf66057307b_875c7d02961ba5720596e10e2c5e2c5e, store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:39,860 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111462609fa58a4bc9a040ccf66057307b_875c7d02961ba5720596e10e2c5e2c5e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:39,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742199_1375 (size=12847) 2024-11-11T02:32:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742201_1377 (size=4469) 2024-11-11T02:32:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742200_1376 (size=9914) 2024-11-11T02:32:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T02:32:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,268 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#A#compaction#310 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:40,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,269 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/5b85dff4051947289c7353b145e18cee is 175, key is test_row_0/A:col10/1731292357001/Put/seqid=0 2024-11-11T02:32:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,285 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/8fb249b5707e4dd19be3090f1d03bb62 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/8fb249b5707e4dd19be3090f1d03bb62 2024-11-11T02:32:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,290 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/B of 875c7d02961ba5720596e10e2c5e2c5e into 8fb249b5707e4dd19be3090f1d03bb62(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:40,290 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:40,290 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/B, priority=13, startTime=1731292359830; duration=0sec 2024-11-11T02:32:40,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742202_1378 (size=31801) 2024-11-11T02:32:40,290 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:40,290 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:B 2024-11-11T02:32:40,290 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,292 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:40,292 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/C is initiating minor compaction (all files) 2024-11-11T02:32:40,292 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/C in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,292 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/5fffa3b669f24723a9c46893aa5fc21c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/41221cc410d249e0a359bee9e0acc4df, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c3a1dba2432546209521b854be653b4e] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=36.2 K 2024-11-11T02:32:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,293 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fffa3b669f24723a9c46893aa5fc21c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731292354368 2024-11-11T02:32:40,293 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 41221cc410d249e0a359bee9e0acc4df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731292354718 2024-11-11T02:32:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,294 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting c3a1dba2432546209521b854be653b4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731292357001 2024-11-11T02:32:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,299 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/5b85dff4051947289c7353b145e18cee as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/5b85dff4051947289c7353b145e18cee 2024-11-11T02:32:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,302 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114565a98f5f9643aba22cdc8505795f28_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114565a98f5f9643aba22cdc8505795f28_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/f666ca3af37542588775ac55b4648021, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,304 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/A of 875c7d02961ba5720596e10e2c5e2c5e into 5b85dff4051947289c7353b145e18cee(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:40,304 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:40,304 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/A, priority=13, startTime=1731292359830; duration=0sec 2024-11-11T02:32:40,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/f666ca3af37542588775ac55b4648021 is 175, key is test_row_0/A:col10/1731292358192/Put/seqid=0 2024-11-11T02:32:40,304 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:40,304 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:A 2024-11-11T02:32:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,306 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#C#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,306 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/3f4e00c47df248869620b922882d4f23 is 50, key is test_row_0/C:col10/1731292357001/Put/seqid=0 2024-11-11T02:32:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742203_1379 (size=22561) 2024-11-11T02:32:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,326 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/f666ca3af37542588775ac55b4648021 2024-11-11T02:32:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/9ebb9b231fab4d3bb8c1b443a49dbc3c is 50, key is test_row_0/B:col10/1731292358192/Put/seqid=0 2024-11-11T02:32:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742204_1380 (size=12847) 2024-11-11T02:32:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,345 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/3f4e00c47df248869620b922882d4f23 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/3f4e00c47df248869620b922882d4f23 2024-11-11T02:32:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,350 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/C of 875c7d02961ba5720596e10e2c5e2c5e into 3f4e00c47df248869620b922882d4f23(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:40,350 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,350 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/C, priority=13, startTime=1731292359830; duration=0sec 2024-11-11T02:32:40,350 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:40,350 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:C 2024-11-11T02:32:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742205_1381 (size=9857) 2024-11-11T02:32:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:40,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. as already flushing 2024-11-11T02:32:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:40,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292360438, max timestamp=1731292360458Num uniques 859; 2024-11-11T02:32:40,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:40,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292360438, max timestamp=1731292360458Num uniques 859; 2024-11-11T02:32:40,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:40,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292360438, max timestamp=1731292360458Num uniques 859; 2024-11-11T02:32:40,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:40,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731292360499, max timestamp=1731292360511Num uniques 1068; 2024-11-11T02:32:40,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T02:32:40,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292420503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:40,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731292360499, max timestamp=1731292360511Num uniques 1068; 2024-11-11T02:32:40,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731292360499, max timestamp=1731292360511Num uniques 1068; 2024-11-11T02:32:40,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T02:32:40,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T02:32:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:40,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731292360545, max timestamp=1731292360570Num uniques 1077; 2024-11-11T02:32:40,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-11T02:32:40,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731292360545, max timestamp=1731292360570Num uniques 1077; 2024-11-11T02:32:40,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-11T02:32:40,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731292360545, max timestamp=1731292360570Num uniques 1077; 2024-11-11T02:32:40,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-11T02:32:40,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731292360592, max timestamp=1731292360631Num uniques 1086; 2024-11-11T02:32:40,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-11T02:32:40,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731292360592, max timestamp=1731292360631Num uniques 1086; 2024-11-11T02:32:40,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-11T02:32:40,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731292360592, max timestamp=1731292360631Num uniques 1086; 2024-11-11T02:32:40,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-11T02:32:40,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292420678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:40,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292420683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:40,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292420687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:40,760 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/9ebb9b231fab4d3bb8c1b443a49dbc3c 2024-11-11T02:32:40,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/8d04f621d1734501ad1704489768b8b7 is 50, key is test_row_0/C:col10/1731292358192/Put/seqid=0 2024-11-11T02:32:40,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742206_1382 (size=9857) 2024-11-11T02:32:40,770 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/8d04f621d1734501ad1704489768b8b7 2024-11-11T02:32:40,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/f666ca3af37542588775ac55b4648021 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f666ca3af37542588775ac55b4648021 2024-11-11T02:32:40,777 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f666ca3af37542588775ac55b4648021, entries=100, sequenceid=277, filesize=22.0 K 2024-11-11T02:32:40,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/9ebb9b231fab4d3bb8c1b443a49dbc3c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/9ebb9b231fab4d3bb8c1b443a49dbc3c 2024-11-11T02:32:40,785 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/9ebb9b231fab4d3bb8c1b443a49dbc3c, entries=100, sequenceid=277, filesize=9.6 K 2024-11-11T02:32:40,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/8d04f621d1734501ad1704489768b8b7 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/8d04f621d1734501ad1704489768b8b7 2024-11-11T02:32:40,789 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/8d04f621d1734501ad1704489768b8b7, entries=100, sequenceid=277, filesize=9.6 K 2024-11-11T02:32:40,789 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 875c7d02961ba5720596e10e2c5e2c5e in 955ms, sequenceid=277, compaction requested=false 2024-11-11T02:32:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-11T02:32:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-11T02:32:40,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-11T02:32:40,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8710 sec 2024-11-11T02:32:40,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.8760 sec 2024-11-11T02:32:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:40,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=154.31 KB heapSize=433.55 KB 2024-11-11T02:32:40,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731292360672, max timestamp=1731292360804Num uniques 1145; 2024-11-11T02:32:40,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-11T02:32:40,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731292360672, max timestamp=1731292360804Num uniques 1145; 2024-11-11T02:32:40,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-11T02:32:40,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731292360672, max timestamp=1731292360804Num uniques 1145; 2024-11-11T02:32:40,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-11T02:32:40,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:40,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:40,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:40,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:40,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:40,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:40,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111188cf338048e9453a87b3d08219765778_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292360804/Put/seqid=0 2024-11-11T02:32:40,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742207_1383 (size=17534) 2024-11-11T02:32:40,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292360852, max timestamp=1731292360875Num uniques 859; 2024-11-11T02:32:40,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:32:40,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292360852, max timestamp=1731292360875Num uniques 859; 2024-11-11T02:32:40,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292360852, max timestamp=1731292360875Num uniques 859; 2024-11-11T02:32:40,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:32:40,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:32:40,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292420932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:40,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292420933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:40,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292420935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:40,970 INFO [Thread-1437 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1437. 2024-11-11T02:32:40,970 DEBUG [Thread-1437 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:40,970 DEBUG [Thread-1437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:40,970 DEBUG [Thread-1437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:40,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:40,972 INFO [Thread-1439 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1439. 2024-11-11T02:32:40,972 DEBUG [Thread-1439 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:40,972 DEBUG [Thread-1439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:40,972 DEBUG [Thread-1439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:40,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:40,975 INFO [Thread-1435 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1435. 2024-11-11T02:32:40,975 DEBUG [Thread-1435 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:40,975 DEBUG [Thread-1435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:40,975 DEBUG [Thread-1435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:40,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:40,982 INFO [Thread-1443 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1443. 2024-11-11T02:32:40,982 DEBUG [Thread-1443 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:40,982 DEBUG [Thread-1443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:40,983 DEBUG [Thread-1443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:40,983 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:40,985 INFO [Thread-1441 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1441. 2024-11-11T02:32:40,985 DEBUG [Thread-1441 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:40,985 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:40,985 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:40,985 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:41,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:41,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292421056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292421056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:41,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292421056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T02:32:41,066 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:41,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:41,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51168 deadline: 1731292421236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:41,238 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in 875c7d02961ba5720596e10e2c5e2c5e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:32:41,239 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:41,245 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111188cf338048e9453a87b3d08219765778_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111188cf338048e9453a87b3d08219765778_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:41,247 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/5a20ec5b401a489591544e54275f3a47, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:41,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/5a20ec5b401a489591544e54275f3a47 is 175, key is test_row_0/A:col10/1731292360804/Put/seqid=0 2024-11-11T02:32:41,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742208_1384 (size=48639) 2024-11-11T02:32:41,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:41,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:41,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:41,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292421265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:41,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292421265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:41,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292421265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:41,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:41,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:41,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:41,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292421576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:41,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:41,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292421577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292421577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:41,653 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/5a20ec5b401a489591544e54275f3a47 2024-11-11T02:32:41,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/2c462d80d2c24bd1b645822a5836e475 is 50, key is test_row_0/B:col10/1731292360804/Put/seqid=0 2024-11-11T02:32:41,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742209_1385 (size=12301) 2024-11-11T02:32:42,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/2c462d80d2c24bd1b645822a5836e475 2024-11-11T02:32:42,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/f4671093521b4ce98abe62e3fb68dbcc is 50, key is test_row_0/C:col10/1731292360804/Put/seqid=0 2024-11-11T02:32:42,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742210_1386 (size=12301) 2024-11-11T02:32:42,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:42,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:42,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51184 deadline: 1731292422086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51198 deadline: 1731292422086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51188 deadline: 1731292422086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:32:42,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/f4671093521b4ce98abe62e3fb68dbcc 2024-11-11T02:32:42,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/5a20ec5b401a489591544e54275f3a47 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/5a20ec5b401a489591544e54275f3a47 2024-11-11T02:32:42,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/5a20ec5b401a489591544e54275f3a47, entries=250, sequenceid=370, filesize=47.5 K 2024-11-11T02:32:42,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/2c462d80d2c24bd1b645822a5836e475 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/2c462d80d2c24bd1b645822a5836e475 2024-11-11T02:32:42,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/2c462d80d2c24bd1b645822a5836e475, entries=150, sequenceid=370, filesize=12.0 K 2024-11-11T02:32:42,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/f4671093521b4ce98abe62e3fb68dbcc as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/f4671093521b4ce98abe62e3fb68dbcc 2024-11-11T02:32:42,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/f4671093521b4ce98abe62e3fb68dbcc, entries=150, sequenceid=370, filesize=12.0 K 2024-11-11T02:32:42,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~82.73 KB/84720, currentSize=154.31 KB/158010 for 875c7d02961ba5720596e10e2c5e2c5e in 1696ms, sequenceid=370, compaction requested=true 2024-11-11T02:32:42,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:42,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:32:42,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:42,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:32:42,502 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:42,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:42,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 875c7d02961ba5720596e10e2c5e2c5e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:32:42,502 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:42,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:42,503 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:42,503 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:42,503 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/A is initiating minor compaction (all files) 2024-11-11T02:32:42,503 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/B is initiating minor compaction (all files) 2024-11-11T02:32:42,503 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/B in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:42,503 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/A in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:42,503 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/5b85dff4051947289c7353b145e18cee, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f666ca3af37542588775ac55b4648021, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/5a20ec5b401a489591544e54275f3a47] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=100.6 K 2024-11-11T02:32:42,503 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/8fb249b5707e4dd19be3090f1d03bb62, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/9ebb9b231fab4d3bb8c1b443a49dbc3c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/2c462d80d2c24bd1b645822a5836e475] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=34.2 K 2024-11-11T02:32:42,503 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:42,503 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/5b85dff4051947289c7353b145e18cee, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f666ca3af37542588775ac55b4648021, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/5a20ec5b401a489591544e54275f3a47] 2024-11-11T02:32:42,503 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fb249b5707e4dd19be3090f1d03bb62, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731292357001 2024-11-11T02:32:42,503 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b85dff4051947289c7353b145e18cee, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731292357001 2024-11-11T02:32:42,504 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ebb9b231fab4d3bb8c1b443a49dbc3c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731292358188 2024-11-11T02:32:42,504 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting f666ca3af37542588775ac55b4648021, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731292358188 2024-11-11T02:32:42,504 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c462d80d2c24bd1b645822a5836e475, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731292360672 2024-11-11T02:32:42,504 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a20ec5b401a489591544e54275f3a47, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731292360672 2024-11-11T02:32:42,508 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:42,509 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#B#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:42,509 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111c189d9bb0fc1403dac9a58c11679462f_875c7d02961ba5720596e10e2c5e2c5e store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:42,510 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/529e94c2d34045cb89b725f7a44e7b22 is 50, key is test_row_0/B:col10/1731292360804/Put/seqid=0 2024-11-11T02:32:42,512 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111c189d9bb0fc1403dac9a58c11679462f_875c7d02961ba5720596e10e2c5e2c5e, store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:42,512 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c189d9bb0fc1403dac9a58c11679462f_875c7d02961ba5720596e10e2c5e2c5e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:42,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742211_1387 (size=12949) 2024-11-11T02:32:42,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742212_1388 (size=4469) 2024-11-11T02:32:42,917 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#A#compaction#319 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:42,917 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/e8fc14182a864abd88d37c75a3159992 is 175, key is test_row_0/A:col10/1731292360804/Put/seqid=0 2024-11-11T02:32:42,919 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/529e94c2d34045cb89b725f7a44e7b22 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/529e94c2d34045cb89b725f7a44e7b22 2024-11-11T02:32:42,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742213_1389 (size=31903) 2024-11-11T02:32:42,925 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/B of 875c7d02961ba5720596e10e2c5e2c5e into 529e94c2d34045cb89b725f7a44e7b22(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:42,925 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:42,925 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/B, priority=13, startTime=1731292362502; duration=0sec 2024-11-11T02:32:42,925 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:42,925 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:B 2024-11-11T02:32:42,926 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:42,927 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:42,927 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): 875c7d02961ba5720596e10e2c5e2c5e/C is initiating minor compaction (all files) 2024-11-11T02:32:42,927 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 875c7d02961ba5720596e10e2c5e2c5e/C in TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:42,927 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/3f4e00c47df248869620b922882d4f23, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/8d04f621d1734501ad1704489768b8b7, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/f4671093521b4ce98abe62e3fb68dbcc] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp, totalSize=34.2 K 2024-11-11T02:32:42,927 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f4e00c47df248869620b922882d4f23, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731292357001 2024-11-11T02:32:42,928 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d04f621d1734501ad1704489768b8b7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731292358188 2024-11-11T02:32:42,928 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting f4671093521b4ce98abe62e3fb68dbcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731292360672 2024-11-11T02:32:42,936 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 875c7d02961ba5720596e10e2c5e2c5e#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:42,937 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/b1fa6bde07824a5c8c86ab2b4696b6d1 is 50, key is test_row_0/C:col10/1731292360804/Put/seqid=0 2024-11-11T02:32:42,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742214_1390 (size=12949) 2024-11-11T02:32:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:43,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=167.72 KB heapSize=471.23 KB 2024-11-11T02:32:43,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:43,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:32:43,102 INFO [Thread-1432 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1432. 2024-11-11T02:32:43,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:43,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:32:43,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:43,103 DEBUG [Thread-1432 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:43,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:32:43,103 INFO [Thread-1426 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1426. 2024-11-11T02:32:43,103 DEBUG [Thread-1426 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:43,103 DEBUG [Thread-1432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:43,103 DEBUG [Thread-1426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:43,103 DEBUG [Thread-1426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:43,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:43,104 DEBUG [Thread-1432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:43,104 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:43,107 INFO [Thread-1424 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1424. 2024-11-11T02:32:43,107 DEBUG [Thread-1424 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:43,107 DEBUG [Thread-1424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:43,107 DEBUG [Thread-1424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:43,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:43,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118add2d7bb48f4ce493c8f85a084ac5bf_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292363097/Put/seqid=0 2024-11-11T02:32:43,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742215_1391 (size=12454) 2024-11-11T02:32:43,334 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/e8fc14182a864abd88d37c75a3159992 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/e8fc14182a864abd88d37c75a3159992 2024-11-11T02:32:43,339 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/A of 875c7d02961ba5720596e10e2c5e2c5e into e8fc14182a864abd88d37c75a3159992(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:43,339 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:43,339 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/A, priority=13, startTime=1731292362502; duration=0sec 2024-11-11T02:32:43,339 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:43,340 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:A 2024-11-11T02:32:43,347 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/b1fa6bde07824a5c8c86ab2b4696b6d1 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/b1fa6bde07824a5c8c86ab2b4696b6d1 2024-11-11T02:32:43,353 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 875c7d02961ba5720596e10e2c5e2c5e/C of 875c7d02961ba5720596e10e2c5e2c5e into b1fa6bde07824a5c8c86ab2b4696b6d1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:43,353 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:43,353 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., storeName=875c7d02961ba5720596e10e2c5e2c5e/C, priority=13, startTime=1731292362502; duration=0sec 2024-11-11T02:32:43,353 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:43,354 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 875c7d02961ba5720596e10e2c5e2c5e:C 2024-11-11T02:32:43,513 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:43,519 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118add2d7bb48f4ce493c8f85a084ac5bf_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118add2d7bb48f4ce493c8f85a084ac5bf_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:43,520 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/7f25c66543e0432fad567cfd8d6ed49f, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:43,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/7f25c66543e0432fad567cfd8d6ed49f is 175, key is test_row_0/A:col10/1731292363097/Put/seqid=0 2024-11-11T02:32:43,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742216_1392 (size=31255) 2024-11-11T02:32:43,928 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/7f25c66543e0432fad567cfd8d6ed49f 2024-11-11T02:32:43,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/f350602977d14f0e82f9333b7c40c1ec is 50, key is test_row_0/B:col10/1731292363097/Put/seqid=0 2024-11-11T02:32:43,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742217_1393 (size=12301) 2024-11-11T02:32:44,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/f350602977d14f0e82f9333b7c40c1ec 2024-11-11T02:32:44,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/c1cbd7beba5b43acb4ea17d81106d41c is 50, key is test_row_0/C:col10/1731292363097/Put/seqid=0 2024-11-11T02:32:44,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742218_1394 (size=12301) 2024-11-11T02:32:44,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/c1cbd7beba5b43acb4ea17d81106d41c 2024-11-11T02:32:44,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/7f25c66543e0432fad567cfd8d6ed49f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/7f25c66543e0432fad567cfd8d6ed49f 2024-11-11T02:32:44,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/7f25c66543e0432fad567cfd8d6ed49f, entries=150, sequenceid=410, filesize=30.5 K 2024-11-11T02:32:44,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/f350602977d14f0e82f9333b7c40c1ec as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/f350602977d14f0e82f9333b7c40c1ec 2024-11-11T02:32:44,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/f350602977d14f0e82f9333b7c40c1ec, entries=150, sequenceid=410, filesize=12.0 K 2024-11-11T02:32:44,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/c1cbd7beba5b43acb4ea17d81106d41c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c1cbd7beba5b43acb4ea17d81106d41c 2024-11-11T02:32:44,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c1cbd7beba5b43acb4ea17d81106d41c, entries=150, sequenceid=410, filesize=12.0 K 2024-11-11T02:32:44,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~471.19 KB/482496, currentSize=6.71 KB/6870 for 875c7d02961ba5720596e10e2c5e2c5e in 1690ms, sequenceid=410, compaction requested=false 2024-11-11T02:32:44,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 875c7d02961ba5720596e10e2c5e2c5e: 2024-11-11T02:32:45,251 INFO [Thread-1428 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1428. 2024-11-11T02:32:45,251 DEBUG [Thread-1428 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:45,251 DEBUG [Thread-1428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:45,252 DEBUG [Thread-1428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:45,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:46,014 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T02:32:49,439 INFO [Thread-1430 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1430. 2024-11-11T02:32:49,440 DEBUG [Thread-1430 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:32:49,440 DEBUG [Thread-1430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:49,440 DEBUG [Thread-1430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:49,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:49,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T02:32:49,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-11T02:32:49,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-11T02:32:49,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-11T02:32:49,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-11T02:32:49,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-11T02:32:49,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T02:32:49,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T02:32:49,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1111 2024-11-11T02:32:49,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3328 rows 2024-11-11T02:32:49,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1106 2024-11-11T02:32:49,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3311 rows 2024-11-11T02:32:49,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1105 2024-11-11T02:32:49,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3307 rows 2024-11-11T02:32:49,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1097 2024-11-11T02:32:49,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3287 rows 2024-11-11T02:32:49,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1102 2024-11-11T02:32:49,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3303 rows 2024-11-11T02:32:49,441 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:32:49,441 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:32:49,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:49,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:49,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:32:49,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T02:32:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T02:32:49,447 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292369447"}]},"ts":"1731292369447"} 2024-11-11T02:32:49,449 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T02:32:49,449 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T02:32:49,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T02:32:49,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, UNASSIGN}] 2024-11-11T02:32:49,453 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, UNASSIGN 2024-11-11T02:32:49,454 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=875c7d02961ba5720596e10e2c5e2c5e, regionState=CLOSING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:32:49,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, UNASSIGN because future has completed 2024-11-11T02:32:49,456 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T02:32:49,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:32:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T02:32:49,611 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:49,611 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T02:32:49,611 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 875c7d02961ba5720596e10e2c5e2c5e, disabling compactions & flushes 2024-11-11T02:32:49,611 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:49,611 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:49,611 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. after waiting 0 ms 2024-11-11T02:32:49,612 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:49,612 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 875c7d02961ba5720596e10e2c5e2c5e 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-11T02:32:49,612 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=A 2024-11-11T02:32:49,613 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:49,613 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=B 2024-11-11T02:32:49,613 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:49,613 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 875c7d02961ba5720596e10e2c5e2c5e, store=C 2024-11-11T02:32:49,613 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:49,622 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112b068084954f4c7d9aebd74ea47237cf_875c7d02961ba5720596e10e2c5e2c5e is 50, key is test_row_0/A:col10/1731292365248/Put/seqid=0 2024-11-11T02:32:49,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742219_1395 (size=9914) 2024-11-11T02:32:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T02:32:50,028 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:50,032 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112b068084954f4c7d9aebd74ea47237cf_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112b068084954f4c7d9aebd74ea47237cf_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:50,033 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/639ac92309784050a622678f6d641103, store: [table=TestAcidGuarantees family=A region=875c7d02961ba5720596e10e2c5e2c5e] 2024-11-11T02:32:50,033 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/639ac92309784050a622678f6d641103 is 175, key is test_row_0/A:col10/1731292365248/Put/seqid=0 2024-11-11T02:32:50,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742220_1396 (size=22561) 2024-11-11T02:32:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T02:32:50,081 WARN [regionserver/370bc2ade342:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-11T02:32:50,083 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/ec6101a136924812a6c3a037d88091fa, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/cb397d7505394396866b4700b3c4855b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/cc01b7591c16424aa1adcaf8123f2b66, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/ff471b47da6c4e38a357183b4dfa3fdd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/4ccecb5f402b4d2da731a7d274da70c5, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/e2ab3317c9bd484c881cf9e854191919, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/6e5751b919be4a68a4163701d30b64c9, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/1802899eb7ff44c186ea06ef3de569ea, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/35abbde42c384735aec9d5cc04cb552b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f196030c82c64961861aa57e4a505cee, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f73e0b0403b94999b9b9fcea4e696339, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/0200e4d30ed2452c8ba7332838f6f2eb, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/d4205093e4b94d40906fb57b1ddb3ca4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/c15ffc18a9554b16b75b20c3f96d3bb5, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/6f6790d2868049d9be580b7a81fc3f2e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/76d1abb3d1d6453c90098437ebcc8272, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/5b85dff4051947289c7353b145e18cee, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f666ca3af37542588775ac55b4648021, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/5a20ec5b401a489591544e54275f3a47] to archive 2024-11-11T02:32:50,085 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:32:50,089 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/ec6101a136924812a6c3a037d88091fa to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/ec6101a136924812a6c3a037d88091fa 2024-11-11T02:32:50,092 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/cb397d7505394396866b4700b3c4855b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/cb397d7505394396866b4700b3c4855b 2024-11-11T02:32:50,093 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/cc01b7591c16424aa1adcaf8123f2b66 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/cc01b7591c16424aa1adcaf8123f2b66 2024-11-11T02:32:50,095 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/ff471b47da6c4e38a357183b4dfa3fdd to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/ff471b47da6c4e38a357183b4dfa3fdd 2024-11-11T02:32:50,096 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/4ccecb5f402b4d2da731a7d274da70c5 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/4ccecb5f402b4d2da731a7d274da70c5 2024-11-11T02:32:50,098 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/e2ab3317c9bd484c881cf9e854191919 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/e2ab3317c9bd484c881cf9e854191919 2024-11-11T02:32:50,099 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/6e5751b919be4a68a4163701d30b64c9 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/6e5751b919be4a68a4163701d30b64c9 2024-11-11T02:32:50,100 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/1802899eb7ff44c186ea06ef3de569ea to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/1802899eb7ff44c186ea06ef3de569ea 2024-11-11T02:32:50,102 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/35abbde42c384735aec9d5cc04cb552b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/35abbde42c384735aec9d5cc04cb552b 2024-11-11T02:32:50,103 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f196030c82c64961861aa57e4a505cee to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f196030c82c64961861aa57e4a505cee 2024-11-11T02:32:50,104 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f73e0b0403b94999b9b9fcea4e696339 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f73e0b0403b94999b9b9fcea4e696339 2024-11-11T02:32:50,105 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/0200e4d30ed2452c8ba7332838f6f2eb to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/0200e4d30ed2452c8ba7332838f6f2eb 2024-11-11T02:32:50,107 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/d4205093e4b94d40906fb57b1ddb3ca4 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/d4205093e4b94d40906fb57b1ddb3ca4 2024-11-11T02:32:50,109 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/c15ffc18a9554b16b75b20c3f96d3bb5 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/c15ffc18a9554b16b75b20c3f96d3bb5 2024-11-11T02:32:50,110 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/6f6790d2868049d9be580b7a81fc3f2e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/6f6790d2868049d9be580b7a81fc3f2e 2024-11-11T02:32:50,112 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/76d1abb3d1d6453c90098437ebcc8272 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/76d1abb3d1d6453c90098437ebcc8272 2024-11-11T02:32:50,113 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/5b85dff4051947289c7353b145e18cee to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/5b85dff4051947289c7353b145e18cee 2024-11-11T02:32:50,114 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f666ca3af37542588775ac55b4648021 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/f666ca3af37542588775ac55b4648021 2024-11-11T02:32:50,115 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/5a20ec5b401a489591544e54275f3a47 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/5a20ec5b401a489591544e54275f3a47 2024-11-11T02:32:50,119 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/5657c53483a84ecf83a065051cea565e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/8217b7328b284ed79a8debdcc0772995, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/0d064343443b46609fdb490a1b1dc77e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/62c867129a734878aebe78446b657ef6, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/7408731199b34b2ba4ac21c2a8dff69a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/0a78aa756c324ef0a9573e292c646c45, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/e7102085bf7a4968933e5a95477b3524, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/bf801db1df424c0ea7874ce0ec9164b8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/7f72f277114340cebb0aef688f4516e8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/71f1f5cc158d41fa8b60ed0195da0a4f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/30f7d0bccf834d7dbfa1ff0ef188091a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/04deeb84bdbc4f569b3544ed1113085a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/047e0613a1914747ac485f8d0681a6c2, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/ec27cfc7e6cd44259f269dfc22d540e9, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/875838a981624ab9940390e886a0951f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/8fb249b5707e4dd19be3090f1d03bb62, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/ee8d8ca9e61640dc86efe563fff8ab8d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/9ebb9b231fab4d3bb8c1b443a49dbc3c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/2c462d80d2c24bd1b645822a5836e475] to archive 2024-11-11T02:32:50,120 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:32:50,122 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/5657c53483a84ecf83a065051cea565e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/5657c53483a84ecf83a065051cea565e 2024-11-11T02:32:50,124 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/8217b7328b284ed79a8debdcc0772995 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/8217b7328b284ed79a8debdcc0772995 2024-11-11T02:32:50,125 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/0d064343443b46609fdb490a1b1dc77e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/0d064343443b46609fdb490a1b1dc77e 2024-11-11T02:32:50,126 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/62c867129a734878aebe78446b657ef6 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/62c867129a734878aebe78446b657ef6 2024-11-11T02:32:50,128 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/7408731199b34b2ba4ac21c2a8dff69a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/7408731199b34b2ba4ac21c2a8dff69a 2024-11-11T02:32:50,129 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/0a78aa756c324ef0a9573e292c646c45 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/0a78aa756c324ef0a9573e292c646c45 2024-11-11T02:32:50,131 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/e7102085bf7a4968933e5a95477b3524 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/e7102085bf7a4968933e5a95477b3524 2024-11-11T02:32:50,132 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/bf801db1df424c0ea7874ce0ec9164b8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/bf801db1df424c0ea7874ce0ec9164b8 2024-11-11T02:32:50,133 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/7f72f277114340cebb0aef688f4516e8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/7f72f277114340cebb0aef688f4516e8 2024-11-11T02:32:50,135 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/71f1f5cc158d41fa8b60ed0195da0a4f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/71f1f5cc158d41fa8b60ed0195da0a4f 2024-11-11T02:32:50,136 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/30f7d0bccf834d7dbfa1ff0ef188091a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/30f7d0bccf834d7dbfa1ff0ef188091a 2024-11-11T02:32:50,138 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/04deeb84bdbc4f569b3544ed1113085a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/04deeb84bdbc4f569b3544ed1113085a 2024-11-11T02:32:50,140 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/047e0613a1914747ac485f8d0681a6c2 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/047e0613a1914747ac485f8d0681a6c2 2024-11-11T02:32:50,141 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/ec27cfc7e6cd44259f269dfc22d540e9 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/ec27cfc7e6cd44259f269dfc22d540e9 2024-11-11T02:32:50,142 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/875838a981624ab9940390e886a0951f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/875838a981624ab9940390e886a0951f 2024-11-11T02:32:50,144 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/8fb249b5707e4dd19be3090f1d03bb62 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/8fb249b5707e4dd19be3090f1d03bb62 2024-11-11T02:32:50,146 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/ee8d8ca9e61640dc86efe563fff8ab8d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/ee8d8ca9e61640dc86efe563fff8ab8d 2024-11-11T02:32:50,147 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/9ebb9b231fab4d3bb8c1b443a49dbc3c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/9ebb9b231fab4d3bb8c1b443a49dbc3c 2024-11-11T02:32:50,148 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/2c462d80d2c24bd1b645822a5836e475 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/2c462d80d2c24bd1b645822a5836e475 2024-11-11T02:32:50,152 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/49f952f6b34f4267a770b23460c94069, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/b1cf6baa2d7641ff9090d652caa2073c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/1cf3d5aa891b4887a17373ffa6f7d7a8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/1a7f8dc4bb2b433898a4d87a1e1a1255, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/76088e198cde4188be9349daaf8dd500, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/697188eddcfb4407a3a8dc4c754f94d1, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/14b2a7f6251a45088afc2724a2330e3b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/804192b99a6b40b5ab232e09f6c3c432, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/55dcd18aaa764581827bead7af18312f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/51ca194f218343f5ac819ddd2d628432, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/df2f33c054c54620a58b8234cf7ea797, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/a124b6a3d9184650ab91e8589e5414d9, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/5fffa3b669f24723a9c46893aa5fc21c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c276ad5891b34651843ca643940f503a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/41221cc410d249e0a359bee9e0acc4df, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/3f4e00c47df248869620b922882d4f23, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c3a1dba2432546209521b854be653b4e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/8d04f621d1734501ad1704489768b8b7, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/f4671093521b4ce98abe62e3fb68dbcc] to archive 2024-11-11T02:32:50,153 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:32:50,155 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/49f952f6b34f4267a770b23460c94069 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/49f952f6b34f4267a770b23460c94069 2024-11-11T02:32:50,156 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/b1cf6baa2d7641ff9090d652caa2073c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/b1cf6baa2d7641ff9090d652caa2073c 2024-11-11T02:32:50,158 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/1cf3d5aa891b4887a17373ffa6f7d7a8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/1cf3d5aa891b4887a17373ffa6f7d7a8 2024-11-11T02:32:50,159 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/1a7f8dc4bb2b433898a4d87a1e1a1255 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/1a7f8dc4bb2b433898a4d87a1e1a1255 2024-11-11T02:32:50,160 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/76088e198cde4188be9349daaf8dd500 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/76088e198cde4188be9349daaf8dd500 2024-11-11T02:32:50,162 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/697188eddcfb4407a3a8dc4c754f94d1 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/697188eddcfb4407a3a8dc4c754f94d1 2024-11-11T02:32:50,163 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/14b2a7f6251a45088afc2724a2330e3b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/14b2a7f6251a45088afc2724a2330e3b 2024-11-11T02:32:50,164 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/804192b99a6b40b5ab232e09f6c3c432 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/804192b99a6b40b5ab232e09f6c3c432 2024-11-11T02:32:50,165 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/55dcd18aaa764581827bead7af18312f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/55dcd18aaa764581827bead7af18312f 2024-11-11T02:32:50,166 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/51ca194f218343f5ac819ddd2d628432 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/51ca194f218343f5ac819ddd2d628432 2024-11-11T02:32:50,167 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/df2f33c054c54620a58b8234cf7ea797 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/df2f33c054c54620a58b8234cf7ea797 2024-11-11T02:32:50,168 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/a124b6a3d9184650ab91e8589e5414d9 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/a124b6a3d9184650ab91e8589e5414d9 2024-11-11T02:32:50,169 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/5fffa3b669f24723a9c46893aa5fc21c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/5fffa3b669f24723a9c46893aa5fc21c 2024-11-11T02:32:50,170 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c276ad5891b34651843ca643940f503a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c276ad5891b34651843ca643940f503a 2024-11-11T02:32:50,171 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/41221cc410d249e0a359bee9e0acc4df to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/41221cc410d249e0a359bee9e0acc4df 2024-11-11T02:32:50,172 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/3f4e00c47df248869620b922882d4f23 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/3f4e00c47df248869620b922882d4f23 2024-11-11T02:32:50,174 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c3a1dba2432546209521b854be653b4e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c3a1dba2432546209521b854be653b4e 2024-11-11T02:32:50,175 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/8d04f621d1734501ad1704489768b8b7 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/8d04f621d1734501ad1704489768b8b7 2024-11-11T02:32:50,176 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/f4671093521b4ce98abe62e3fb68dbcc to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/f4671093521b4ce98abe62e3fb68dbcc 2024-11-11T02:32:50,438 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=418, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/639ac92309784050a622678f6d641103 2024-11-11T02:32:50,450 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/be7d622d2d8845e496caabd74c839ed8 is 50, key is test_row_0/B:col10/1731292365248/Put/seqid=0 2024-11-11T02:32:50,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742221_1397 (size=9857) 2024-11-11T02:32:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T02:32:50,856 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/be7d622d2d8845e496caabd74c839ed8 2024-11-11T02:32:50,870 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/ea076cc61e0b41758268b655089b267d is 50, key is test_row_0/C:col10/1731292365248/Put/seqid=0 2024-11-11T02:32:50,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742222_1398 (size=9857) 2024-11-11T02:32:51,275 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/ea076cc61e0b41758268b655089b267d 2024-11-11T02:32:51,285 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/A/639ac92309784050a622678f6d641103 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/639ac92309784050a622678f6d641103 2024-11-11T02:32:51,289 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/639ac92309784050a622678f6d641103, entries=100, sequenceid=418, filesize=22.0 K 2024-11-11T02:32:51,290 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/B/be7d622d2d8845e496caabd74c839ed8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/be7d622d2d8845e496caabd74c839ed8 2024-11-11T02:32:51,293 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/be7d622d2d8845e496caabd74c839ed8, entries=100, sequenceid=418, filesize=9.6 K 2024-11-11T02:32:51,294 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/.tmp/C/ea076cc61e0b41758268b655089b267d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/ea076cc61e0b41758268b655089b267d 2024-11-11T02:32:51,297 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/ea076cc61e0b41758268b655089b267d, entries=100, sequenceid=418, filesize=9.6 K 2024-11-11T02:32:51,298 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 875c7d02961ba5720596e10e2c5e2c5e in 1686ms, sequenceid=418, compaction requested=true 2024-11-11T02:32:51,302 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/recovered.edits/421.seqid, newMaxSeqId=421, maxSeqId=4 2024-11-11T02:32:51,303 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. 2024-11-11T02:32:51,303 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 875c7d02961ba5720596e10e2c5e2c5e: Waiting for close lock at 1731292369611Running coprocessor pre-close hooks at 1731292369611Disabling compacts and flushes for region at 1731292369611Disabling writes for close at 1731292369611Obtaining lock to block concurrent updates at 1731292369612 (+1 ms)Preparing flush snapshotting stores in 875c7d02961ba5720596e10e2c5e2c5e at 1731292369612Finished memstore snapshotting TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731292369613 (+1 ms)Flushing stores of TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e. at 1731292369614 (+1 ms)Flushing 875c7d02961ba5720596e10e2c5e2c5e/A: creating writer at 1731292369615 (+1 ms)Flushing mob file 875c7d02961ba5720596e10e2c5e2c5e/A: appending metadata at 1731292369622 (+7 ms)Flushing mob file 875c7d02961ba5720596e10e2c5e2c5e/A: closing flushed file at 1731292369622Flushing 875c7d02961ba5720596e10e2c5e2c5e/A: appending metadata at 1731292370033 (+411 ms)Flushing 875c7d02961ba5720596e10e2c5e2c5e/A: closing flushed file at 1731292370033Flushing 875c7d02961ba5720596e10e2c5e2c5e/B: creating writer at 1731292370447 (+414 ms)Flushing 875c7d02961ba5720596e10e2c5e2c5e/B: appending metadata at 1731292370450 (+3 ms)Flushing 875c7d02961ba5720596e10e2c5e2c5e/B: closing flushed file at 1731292370450Flushing 875c7d02961ba5720596e10e2c5e2c5e/C: creating writer at 1731292370865 (+415 ms)Flushing 875c7d02961ba5720596e10e2c5e2c5e/C: appending metadata at 1731292370869 (+4 ms)Flushing 875c7d02961ba5720596e10e2c5e2c5e/C: closing flushed file at 1731292370869Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cfed4ea: reopening flushed file at 1731292371284 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57654ef1: reopening flushed file at 1731292371289 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ac8d755: reopening flushed file at 1731292371293 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 875c7d02961ba5720596e10e2c5e2c5e in 1686ms, sequenceid=418, compaction requested=true at 1731292371298 (+5 ms)Writing region close event to WAL at 1731292371299 (+1 ms)Running coprocessor post-close hooks at 1731292371302 (+3 ms)Closed at 1731292371302 2024-11-11T02:32:51,304 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,305 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=875c7d02961ba5720596e10e2c5e2c5e, regionState=CLOSED 2024-11-11T02:32:51,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:32:51,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-11T02:32:51,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 875c7d02961ba5720596e10e2c5e2c5e, server=370bc2ade342,38037,1731292247443 in 1.8510 sec 2024-11-11T02:32:51,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-11T02:32:51,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=875c7d02961ba5720596e10e2c5e2c5e, UNASSIGN in 1.8560 sec 2024-11-11T02:32:51,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-11T02:32:51,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8600 sec 2024-11-11T02:32:51,312 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292371312"}]},"ts":"1731292371312"} 2024-11-11T02:32:51,313 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T02:32:51,313 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T02:32:51,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8710 sec 2024-11-11T02:32:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T02:32:51,596 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:51,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T02:32:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:51,602 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T02:32:51,604 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:51,607 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,610 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/recovered.edits] 2024-11-11T02:32:51,612 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/639ac92309784050a622678f6d641103 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/639ac92309784050a622678f6d641103 2024-11-11T02:32:51,613 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/7f25c66543e0432fad567cfd8d6ed49f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/7f25c66543e0432fad567cfd8d6ed49f 2024-11-11T02:32:51,613 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/e8fc14182a864abd88d37c75a3159992 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/A/e8fc14182a864abd88d37c75a3159992 2024-11-11T02:32:51,615 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/529e94c2d34045cb89b725f7a44e7b22 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/529e94c2d34045cb89b725f7a44e7b22 2024-11-11T02:32:51,615 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/be7d622d2d8845e496caabd74c839ed8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/be7d622d2d8845e496caabd74c839ed8 2024-11-11T02:32:51,616 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/f350602977d14f0e82f9333b7c40c1ec to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/B/f350602977d14f0e82f9333b7c40c1ec 2024-11-11T02:32:51,617 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/b1fa6bde07824a5c8c86ab2b4696b6d1 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/b1fa6bde07824a5c8c86ab2b4696b6d1 2024-11-11T02:32:51,618 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c1cbd7beba5b43acb4ea17d81106d41c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/c1cbd7beba5b43acb4ea17d81106d41c 2024-11-11T02:32:51,619 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/ea076cc61e0b41758268b655089b267d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/C/ea076cc61e0b41758268b655089b267d 2024-11-11T02:32:51,620 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/recovered.edits/421.seqid to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e/recovered.edits/421.seqid 2024-11-11T02:32:51,621 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,621 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T02:32:51,621 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T02:32:51,622 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T02:32:51,625 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110cf1dd3ad2184791b4db3c88480383b6_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110cf1dd3ad2184791b4db3c88480383b6_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,626 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110fc69b8396d04738b3db3f7c71a8d77b_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110fc69b8396d04738b3db3f7c71a8d77b_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,627 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111140aa1b3dfaf40308e2a6618612634fa_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111140aa1b3dfaf40308e2a6618612634fa_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,628 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112b068084954f4c7d9aebd74ea47237cf_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112b068084954f4c7d9aebd74ea47237cf_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,629 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113b9b38d891e14020a4594306051386b5_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113b9b38d891e14020a4594306051386b5_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,630 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114565a98f5f9643aba22cdc8505795f28_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114565a98f5f9643aba22cdc8505795f28_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,630 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146ae9ff764ad4d4f8ef5f13f85dad705_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146ae9ff764ad4d4f8ef5f13f85dad705_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,631 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114a0d927ce9cb48198f1f8eeee99ba9f4_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114a0d927ce9cb48198f1f8eeee99ba9f4_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,632 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111170a203d90ef34d14b318f2d127de542c_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111170a203d90ef34d14b318f2d127de542c_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,632 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111187f3c322e3f141988f692273f62ecfea_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111187f3c322e3f141988f692273f62ecfea_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,633 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111188cf338048e9453a87b3d08219765778_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111188cf338048e9453a87b3d08219765778_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,634 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118add2d7bb48f4ce493c8f85a084ac5bf_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118add2d7bb48f4ce493c8f85a084ac5bf_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,634 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c5cb647e404e4c4d92e367a8c2dd9e30_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c5cb647e404e4c4d92e367a8c2dd9e30_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,635 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d9f5473adce8496f8e14937fab9965c0_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d9f5473adce8496f8e14937fab9965c0_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,635 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e69e878ce4aa403a86719cdb8edf38c6_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e69e878ce4aa403a86719cdb8edf38c6_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,636 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ed9f4e5d8bbb4b48a8f9a9a4ebbad4ef_875c7d02961ba5720596e10e2c5e2c5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ed9f4e5d8bbb4b48a8f9a9a4ebbad4ef_875c7d02961ba5720596e10e2c5e2c5e 2024-11-11T02:32:51,636 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T02:32:51,638 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:51,640 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T02:32:51,641 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T02:32:51,642 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:51,642 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T02:32:51,642 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731292371642"}]},"ts":"9223372036854775807"} 2024-11-11T02:32:51,644 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T02:32:51,644 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 875c7d02961ba5720596e10e2c5e2c5e, NAME => 'TestAcidGuarantees,,1731292337803.875c7d02961ba5720596e10e2c5e2c5e.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T02:32:51,644 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T02:32:51,644 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731292371644"}]},"ts":"9223372036854775807"} 2024-11-11T02:32:51,645 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T02:32:51,646 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:51,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-11T02:32:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T02:32:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T02:32:51,706 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:51,721 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 249), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=437 (was 623), ProcessCount=11 (was 11), AvailableMemoryMB=5564 (was 5618) 2024-11-11T02:32:51,729 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=437, ProcessCount=11, AvailableMemoryMB=5564 2024-11-11T02:32:51,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T02:32:51,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:32:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T02:32:51,732 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T02:32:51,732 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:51,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-11T02:32:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-11T02:32:51,733 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T02:32:51,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742223_1399 (size=1061) 2024-11-11T02:32:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-11T02:32:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-11T02:32:52,145 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe 2024-11-11T02:32:52,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742224_1400 (size=53) 2024-11-11T02:32:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-11T02:32:52,557 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:32:52,557 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c9be960c11aae9a500eccb669509dc81, disabling compactions & flushes 2024-11-11T02:32:52,557 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:52,557 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:52,557 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. after waiting 0 ms 2024-11-11T02:32:52,557 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:52,557 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:52,557 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c9be960c11aae9a500eccb669509dc81: Waiting for close lock at 1731292372557Disabling compacts and flushes for region at 1731292372557Disabling writes for close at 1731292372557Writing region close event to WAL at 1731292372557Closed at 1731292372557 2024-11-11T02:32:52,561 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T02:32:52,561 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731292372561"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731292372561"}]},"ts":"1731292372561"} 2024-11-11T02:32:52,565 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T02:32:52,567 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T02:32:52,567 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292372567"}]},"ts":"1731292372567"} 2024-11-11T02:32:52,569 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T02:32:52,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9be960c11aae9a500eccb669509dc81, ASSIGN}] 2024-11-11T02:32:52,570 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9be960c11aae9a500eccb669509dc81, ASSIGN 2024-11-11T02:32:52,571 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9be960c11aae9a500eccb669509dc81, ASSIGN; state=OFFLINE, location=370bc2ade342,38037,1731292247443; forceNewPlan=false, retain=false 2024-11-11T02:32:52,723 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=c9be960c11aae9a500eccb669509dc81, regionState=OPENING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:32:52,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9be960c11aae9a500eccb669509dc81, ASSIGN because future has completed 2024-11-11T02:32:52,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:32:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-11T02:32:52,889 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:52,890 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:32:52,891 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:52,892 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:32:52,892 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:52,892 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:52,894 INFO [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:52,896 INFO [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:32:52,896 INFO [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9be960c11aae9a500eccb669509dc81 columnFamilyName A 2024-11-11T02:32:52,896 DEBUG [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:52,897 INFO [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] regionserver.HStore(327): Store=c9be960c11aae9a500eccb669509dc81/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:32:52,897 INFO [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:52,898 INFO [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:32:52,899 INFO [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9be960c11aae9a500eccb669509dc81 columnFamilyName B 2024-11-11T02:32:52,899 DEBUG [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:52,899 INFO [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] regionserver.HStore(327): Store=c9be960c11aae9a500eccb669509dc81/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:32:52,899 INFO [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:52,901 INFO [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:32:52,901 INFO [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9be960c11aae9a500eccb669509dc81 columnFamilyName C 2024-11-11T02:32:52,901 DEBUG [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:32:52,902 INFO [StoreOpener-c9be960c11aae9a500eccb669509dc81-1 {}] regionserver.HStore(327): Store=c9be960c11aae9a500eccb669509dc81/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:32:52,902 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:52,902 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:52,903 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:52,904 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:52,905 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:52,905 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:52,906 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:32:52,908 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:52,911 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:32:52,911 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened c9be960c11aae9a500eccb669509dc81; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72738467, jitterRate=0.08388762176036835}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:32:52,911 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:52,912 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for c9be960c11aae9a500eccb669509dc81: Running coprocessor pre-open hook at 1731292372892Writing region info on filesystem at 1731292372892Initializing all the Stores at 1731292372893 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292372893Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292372894 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292372894Cleaning up temporary data from old regions at 1731292372905 (+11 ms)Running coprocessor post-open hooks at 1731292372911 (+6 ms)Region opened successfully at 1731292372912 (+1 ms) 2024-11-11T02:32:52,913 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., pid=118, masterSystemTime=1731292372882 2024-11-11T02:32:52,915 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:52,915 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:52,916 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=c9be960c11aae9a500eccb669509dc81, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:32:52,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:32:52,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-11T02:32:52,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 in 192 msec 2024-11-11T02:32:52,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-11T02:32:52,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9be960c11aae9a500eccb669509dc81, ASSIGN in 353 msec 2024-11-11T02:32:52,925 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T02:32:52,926 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292372925"}]},"ts":"1731292372925"} 2024-11-11T02:32:52,927 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T02:32:52,928 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T02:32:52,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1970 sec 2024-11-11T02:32:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-11T02:32:53,877 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:53,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:53,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:53,883 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:53,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:53,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:53,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f4d9468, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:53,887 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4200ce80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,888 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:53,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:53,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:53,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:53,891 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T02:32:53,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 370bc2ade342,42785,1731292246873 2024-11-11T02:32:53,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@13ba9b2d 2024-11-11T02:32:53,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T02:32:53,893 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T02:32:53,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:53,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:53,895 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f3cc86d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:53,896 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fab821c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:53,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:53,898 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f796d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:53,899 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38ff0db4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:53,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:53,900 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b3d1a2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:53,902 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:53,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f6e3fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:53,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:53,903 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6828d62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:53,904 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dbd98f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:53,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:53,905 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ce87ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:53,906 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:53,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ca50f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:53,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:53,908 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@254d64b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:53,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:53,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@342512bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:53,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:53,911 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a7ba970, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:53,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:53,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a239e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:53,914 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:53,914 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@464a34da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:53,916 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b32646b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:53,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:53,918 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36c37526, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:53,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:53,919 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:53,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d8efbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:32:53,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:32:53,922 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:32:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:32:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:32:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2256880e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:32:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:32:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:32:53,923 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:32:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dade5a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:32:53,924 DEBUG [Thread-1831 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:53,924 DEBUG [Thread-1816 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:53,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:53,924 DEBUG [Thread-1825 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:53,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:53,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:53,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:53,925 DEBUG [Thread-1820 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:53,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-11T02:32:53,925 DEBUG [Thread-1827 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T02:32:53,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:53,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:53,926 DEBUG [Thread-1814 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:53,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:53,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:53,927 DEBUG [Thread-1818 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:32:53,928 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:32:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:53,928 DEBUG [Thread-1822 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:53,929 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:53,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:32:53,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:53,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:53,929 DEBUG [Thread-1833 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:32:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:53,930 DEBUG [Thread-1829 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:32:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:32:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:32:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:32:53,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:53,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:53,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:32:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:32:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:32:53,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:32:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:32:53,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2] 2024-11-11T02:32:53,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:32:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:53,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:32:53,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:53,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:32:53,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:53,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:32:53,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:53,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/154b6caa21f347b29e54ac54d408d875 is 50, key is test_row_0/A:col10/1731292373939/Put/seqid=0 2024-11-11T02:32:53,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742225_1401 (size=16681) 2024-11-11T02:32:53,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292373958, max timestamp=1731292373974Num uniques 859; 2024-11-11T02:32:53,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:32:53,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292373958, max timestamp=1731292373974Num uniques 859; 2024-11-11T02:32:53,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292373958, max timestamp=1731292373974Num uniques 859; 2024-11-11T02:32:53,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:32:53,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:32:54,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292434002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292434002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292434003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292434004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292434016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T02:32:54,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T02:32:54,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:54,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:54,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:54,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292434125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292434125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292434125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292434125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292434126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T02:32:54,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:54,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:54,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:54,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T02:32:54,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292434345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292434345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292434346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292434346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292434346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/154b6caa21f347b29e54ac54d408d875 2024-11-11T02:32:54,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T02:32:54,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:54,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:54,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:54,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/a680df97589c4d889a50b5f4fc2f827d is 50, key is test_row_0/B:col10/1731292373939/Put/seqid=0 2024-11-11T02:32:54,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742226_1402 (size=12001) 2024-11-11T02:32:54,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T02:32:54,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:54,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:54,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:54,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T02:32:54,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292434665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292434665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292434665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292434666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292434666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:54,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:54,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T02:32:54,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:54,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:54,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:54,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/a680df97589c4d889a50b5f4fc2f827d 2024-11-11T02:32:54,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/e33c993aee89496f83f4f1329d231ab3 is 50, key is test_row_0/C:col10/1731292373939/Put/seqid=0 2024-11-11T02:32:54,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742227_1403 (size=12001) 2024-11-11T02:32:54,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T02:32:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:54,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:55,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T02:32:55,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:55,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:55,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:55,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:55,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T02:32:55,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T02:32:55,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:55,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:55,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:55,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:55,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:55,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292435175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:55,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292435176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:55,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292435176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:55,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292435176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:55,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292435185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:55,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/e33c993aee89496f83f4f1329d231ab3 2024-11-11T02:32:55,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/154b6caa21f347b29e54ac54d408d875 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/154b6caa21f347b29e54ac54d408d875 2024-11-11T02:32:55,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/154b6caa21f347b29e54ac54d408d875, entries=250, sequenceid=12, filesize=16.3 K 2024-11-11T02:32:55,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/a680df97589c4d889a50b5f4fc2f827d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/a680df97589c4d889a50b5f4fc2f827d 2024-11-11T02:32:55,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/a680df97589c4d889a50b5f4fc2f827d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T02:32:55,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/e33c993aee89496f83f4f1329d231ab3 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/e33c993aee89496f83f4f1329d231ab3 2024-11-11T02:32:55,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/e33c993aee89496f83f4f1329d231ab3, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T02:32:55,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for c9be960c11aae9a500eccb669509dc81 in 1296ms, sequenceid=12, compaction requested=false 2024-11-11T02:32:55,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:32:55,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T02:32:55,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:55,307 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-11T02:32:55,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:32:55,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:55,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:32:55,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:55,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:32:55,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:55,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/e07af46d29e44407a7f55717eaf55954 is 50, key is test_row_0/A:col10/1731292374000/Put/seqid=0 2024-11-11T02:32:55,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742228_1404 (size=12001) 2024-11-11T02:32:55,718 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/e07af46d29e44407a7f55717eaf55954 2024-11-11T02:32:55,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/3835e6ccd84f4f63885b95cf165f394c is 50, key is test_row_0/B:col10/1731292374000/Put/seqid=0 2024-11-11T02:32:55,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742229_1405 (size=12001) 2024-11-11T02:32:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T02:32:56,131 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/3835e6ccd84f4f63885b95cf165f394c 2024-11-11T02:32:56,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/317bdd62af45488cbe08ea8e158bce0e is 50, key is test_row_0/C:col10/1731292374000/Put/seqid=0 2024-11-11T02:32:56,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742230_1406 (size=12001) 2024-11-11T02:32:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:56,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:56,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292436204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292436205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292436205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292436206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292436210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292436316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292436326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292436326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292436326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292436326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,542 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/317bdd62af45488cbe08ea8e158bce0e 2024-11-11T02:32:56,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292436536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/e07af46d29e44407a7f55717eaf55954 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/e07af46d29e44407a7f55717eaf55954 2024-11-11T02:32:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292436545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,549 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/e07af46d29e44407a7f55717eaf55954, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T02:32:56,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/3835e6ccd84f4f63885b95cf165f394c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/3835e6ccd84f4f63885b95cf165f394c 2024-11-11T02:32:56,553 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/3835e6ccd84f4f63885b95cf165f394c, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T02:32:56,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/317bdd62af45488cbe08ea8e158bce0e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/317bdd62af45488cbe08ea8e158bce0e 2024-11-11T02:32:56,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292436545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292436546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292436546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,556 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/317bdd62af45488cbe08ea8e158bce0e, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T02:32:56,557 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for c9be960c11aae9a500eccb669509dc81 in 1250ms, sequenceid=46, compaction requested=false 2024-11-11T02:32:56,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:32:56,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:56,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-11T02:32:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-11T02:32:56,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-11T02:32:56,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6300 sec 2024-11-11T02:32:56,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.6360 sec 2024-11-11T02:32:56,820 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T02:32:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:56,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T02:32:56,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:32:56,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:56,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:32:56,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:56,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:32:56,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:56,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/7b4f8156a20d4d57a403e4bf6a34cd9d is 50, key is test_row_0/A:col10/1731292376858/Put/seqid=0 2024-11-11T02:32:56,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742231_1407 (size=14341) 2024-11-11T02:32:56,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292436892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292436896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292376890, max timestamp=1731292376903Num uniques 859; 2024-11-11T02:32:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292376890, max timestamp=1731292376903Num uniques 859; 2024-11-11T02:32:56,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:56,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292376890, max timestamp=1731292376903Num uniques 859; 2024-11-11T02:32:56,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:32:56,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292436949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292436950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:56,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292436956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:56,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:56,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292437015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292437015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292437065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292437075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292437085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292437236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292437236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/7b4f8156a20d4d57a403e4bf6a34cd9d 2024-11-11T02:32:57,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/2a041be126844b978303fd123630b87d is 50, key is test_row_0/B:col10/1731292376858/Put/seqid=0 2024-11-11T02:32:57,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742232_1408 (size=12001) 2024-11-11T02:32:57,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292437285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292437295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292437306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292437556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292437556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292437605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292437615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292437626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:57,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/2a041be126844b978303fd123630b87d 2024-11-11T02:32:57,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/e57614c43e4b458bb2f151b47dba1a60 is 50, key is test_row_0/C:col10/1731292376858/Put/seqid=0 2024-11-11T02:32:57,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742233_1409 (size=12001) 2024-11-11T02:32:58,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:58,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292438075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:58,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:58,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:58,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292438075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:58,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:58,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/e57614c43e4b458bb2f151b47dba1a60 2024-11-11T02:32:58,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/7b4f8156a20d4d57a403e4bf6a34cd9d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/7b4f8156a20d4d57a403e4bf6a34cd9d 2024-11-11T02:32:58,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T02:32:58,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/7b4f8156a20d4d57a403e4bf6a34cd9d, entries=200, sequenceid=58, filesize=14.0 K 2024-11-11T02:32:58,096 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:32:58,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:32:58,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/2a041be126844b978303fd123630b87d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/2a041be126844b978303fd123630b87d 2024-11-11T02:32:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-11T02:32:58,098 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:32:58,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T02:32:58,098 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:32:58,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:32:58,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/2a041be126844b978303fd123630b87d, entries=150, sequenceid=58, filesize=11.7 K 2024-11-11T02:32:58,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/e57614c43e4b458bb2f151b47dba1a60 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/e57614c43e4b458bb2f151b47dba1a60 2024-11-11T02:32:58,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/e57614c43e4b458bb2f151b47dba1a60, entries=150, sequenceid=58, filesize=11.7 K 2024-11-11T02:32:58,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for c9be960c11aae9a500eccb669509dc81 in 1245ms, sequenceid=58, compaction requested=true 2024-11-11T02:32:58,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:32:58,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:32:58,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:58,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:32:58,105 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:58,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:58,105 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:58,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:32:58,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:58,105 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:58,105 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:58,105 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/B is initiating minor compaction (all files) 2024-11-11T02:32:58,105 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/A is initiating minor compaction (all files) 2024-11-11T02:32:58,106 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/B in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:58,106 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/A in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:58,106 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/a680df97589c4d889a50b5f4fc2f827d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/3835e6ccd84f4f63885b95cf165f394c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/2a041be126844b978303fd123630b87d] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=35.2 K 2024-11-11T02:32:58,106 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/154b6caa21f347b29e54ac54d408d875, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/e07af46d29e44407a7f55717eaf55954, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/7b4f8156a20d4d57a403e4bf6a34cd9d] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=42.0 K 2024-11-11T02:32:58,106 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting a680df97589c4d889a50b5f4fc2f827d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731292373935 2024-11-11T02:32:58,106 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 154b6caa21f347b29e54ac54d408d875, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731292373932 2024-11-11T02:32:58,106 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting e07af46d29e44407a7f55717eaf55954, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731292374000 2024-11-11T02:32:58,106 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 3835e6ccd84f4f63885b95cf165f394c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731292374000 2024-11-11T02:32:58,106 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b4f8156a20d4d57a403e4bf6a34cd9d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731292376203 2024-11-11T02:32:58,106 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a041be126844b978303fd123630b87d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731292376203 2024-11-11T02:32:58,112 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:58,112 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/44c1905dde4641a38db38cc9eba29cba is 50, key is test_row_0/B:col10/1731292376858/Put/seqid=0 2024-11-11T02:32:58,113 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#A#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:58,113 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/4d7aad04976044618795025c20520ff7 is 50, key is test_row_0/A:col10/1731292376858/Put/seqid=0 2024-11-11T02:32:58,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742234_1410 (size=12104) 2024-11-11T02:32:58,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742235_1411 (size=12104) 2024-11-11T02:32:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:32:58,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-11T02:32:58,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:32:58,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:58,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:32:58,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:58,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:32:58,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:32:58,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/6f991537ddd24b51ba4a0e0b6b32d3e0 is 50, key is test_row_0/A:col10/1731292376954/Put/seqid=0 2024-11-11T02:32:58,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742236_1412 (size=14341) 2024-11-11T02:32:58,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:58,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292438161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:58,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:58,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292438171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:58,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292438172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T02:32:58,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T02:32:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:58,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292438275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:58,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:58,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292438286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:58,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292438296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:58,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:58,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T02:32:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:58,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:58,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T02:32:58,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292438485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:58,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292438506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:58,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:58,522 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/44c1905dde4641a38db38cc9eba29cba as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/44c1905dde4641a38db38cc9eba29cba 2024-11-11T02:32:58,526 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/B of c9be960c11aae9a500eccb669509dc81 into 44c1905dde4641a38db38cc9eba29cba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:58,526 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:32:58,526 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/B, priority=13, startTime=1731292378105; duration=0sec 2024-11-11T02:32:58,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:58,526 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:32:58,526 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:B 2024-11-11T02:32:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292438515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:58,526 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:32:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:58,528 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:32:58,528 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/C is initiating minor compaction (all files) 2024-11-11T02:32:58,528 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/C in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:58,528 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/e33c993aee89496f83f4f1329d231ab3, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/317bdd62af45488cbe08ea8e158bce0e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/e57614c43e4b458bb2f151b47dba1a60] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=35.2 K 2024-11-11T02:32:58,528 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting e33c993aee89496f83f4f1329d231ab3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731292373935 2024-11-11T02:32:58,528 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 317bdd62af45488cbe08ea8e158bce0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731292374000 2024-11-11T02:32:58,529 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting e57614c43e4b458bb2f151b47dba1a60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731292376203 2024-11-11T02:32:58,531 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/4d7aad04976044618795025c20520ff7 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/4d7aad04976044618795025c20520ff7 2024-11-11T02:32:58,534 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/A of c9be960c11aae9a500eccb669509dc81 into 4d7aad04976044618795025c20520ff7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:58,534 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:32:58,534 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/A, priority=13, startTime=1731292378104; duration=0sec 2024-11-11T02:32:58,534 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:58,534 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:A 2024-11-11T02:32:58,536 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#C#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:32:58,536 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/7bffca6d330a4075bdb66f5c5b7431d5 is 50, key is test_row_0/C:col10/1731292376858/Put/seqid=0 2024-11-11T02:32:58,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742237_1413 (size=12104) 2024-11-11T02:32:58,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/6f991537ddd24b51ba4a0e0b6b32d3e0 2024-11-11T02:32:58,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/03d6a811a9d945a4bbcfd817b3c63a0e is 50, key is test_row_0/B:col10/1731292376954/Put/seqid=0 2024-11-11T02:32:58,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742238_1414 (size=12001) 2024-11-11T02:32:58,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T02:32:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:58,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T02:32:58,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:58,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:58,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:58,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T02:32:58,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292438805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:58,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292438825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:58,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292438836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:58,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T02:32:58,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:58,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:58,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:58,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:58,943 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/7bffca6d330a4075bdb66f5c5b7431d5 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/7bffca6d330a4075bdb66f5c5b7431d5 2024-11-11T02:32:58,947 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/C of c9be960c11aae9a500eccb669509dc81 into 7bffca6d330a4075bdb66f5c5b7431d5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:32:58,947 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:32:58,947 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/C, priority=13, startTime=1731292378105; duration=0sec 2024-11-11T02:32:58,947 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:32:58,947 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:C 2024-11-11T02:32:58,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/03d6a811a9d945a4bbcfd817b3c63a0e 2024-11-11T02:32:58,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/55ff723270e3462b81fdb15890892acc is 50, key is test_row_0/C:col10/1731292376954/Put/seqid=0 2024-11-11T02:32:58,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742239_1415 (size=12001) 2024-11-11T02:32:59,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T02:32:59,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:59,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:59,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:59,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:59,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:59,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292439086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:59,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292439095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:59,096 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in c9be960c11aae9a500eccb669509dc81 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:32:59,096 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in c9be960c11aae9a500eccb669509dc81 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:32:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:59,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T02:32:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:59,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T02:32:59,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T02:32:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:32:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:59,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:32:59,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292439325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:59,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:59,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:59,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:59,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292439335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:59,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:59,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:59,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:59,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:32:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292439355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:32:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:32:59,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:32:59,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/55ff723270e3462b81fdb15890892acc 2024-11-11T02:32:59,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/6f991537ddd24b51ba4a0e0b6b32d3e0 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/6f991537ddd24b51ba4a0e0b6b32d3e0 2024-11-11T02:32:59,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/6f991537ddd24b51ba4a0e0b6b32d3e0, entries=200, sequenceid=93, filesize=14.0 K 2024-11-11T02:32:59,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/03d6a811a9d945a4bbcfd817b3c63a0e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/03d6a811a9d945a4bbcfd817b3c63a0e 2024-11-11T02:32:59,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/03d6a811a9d945a4bbcfd817b3c63a0e, entries=150, sequenceid=93, filesize=11.7 K 2024-11-11T02:32:59,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/55ff723270e3462b81fdb15890892acc as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/55ff723270e3462b81fdb15890892acc 2024-11-11T02:32:59,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/55ff723270e3462b81fdb15890892acc, entries=150, sequenceid=93, filesize=11.7 K 2024-11-11T02:32:59,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for c9be960c11aae9a500eccb669509dc81 in 1242ms, sequenceid=93, compaction requested=false 2024-11-11T02:32:59,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:32:59,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T02:32:59,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:59,475 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T02:32:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:32:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:32:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:32:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:32:59,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/cefbb9c85fa54146a450d9b94772e50f is 50, key is test_row_0/A:col10/1731292378170/Put/seqid=0 2024-11-11T02:32:59,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742240_1416 (size=12001) 2024-11-11T02:32:59,487 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/cefbb9c85fa54146a450d9b94772e50f 2024-11-11T02:32:59,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/d241f02a33874a469927c98138b01a7a is 50, key is test_row_0/B:col10/1731292378170/Put/seqid=0 2024-11-11T02:32:59,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742241_1417 (size=12001) 2024-11-11T02:32:59,897 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/d241f02a33874a469927c98138b01a7a 2024-11-11T02:32:59,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/22d37c103f544258a12f651860861ba7 is 50, key is test_row_0/C:col10/1731292378170/Put/seqid=0 2024-11-11T02:32:59,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742242_1418 (size=12001) 2024-11-11T02:32:59,915 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/22d37c103f544258a12f651860861ba7 2024-11-11T02:32:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/cefbb9c85fa54146a450d9b94772e50f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/cefbb9c85fa54146a450d9b94772e50f 2024-11-11T02:32:59,923 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/cefbb9c85fa54146a450d9b94772e50f, entries=150, sequenceid=107, filesize=11.7 K 2024-11-11T02:32:59,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/d241f02a33874a469927c98138b01a7a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/d241f02a33874a469927c98138b01a7a 2024-11-11T02:32:59,927 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/d241f02a33874a469927c98138b01a7a, entries=150, sequenceid=107, filesize=11.7 K 2024-11-11T02:32:59,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/22d37c103f544258a12f651860861ba7 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/22d37c103f544258a12f651860861ba7 2024-11-11T02:32:59,933 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/22d37c103f544258a12f651860861ba7, entries=150, sequenceid=107, filesize=11.7 K 2024-11-11T02:32:59,933 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for c9be960c11aae9a500eccb669509dc81 in 459ms, sequenceid=107, compaction requested=true 2024-11-11T02:32:59,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:32:59,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:32:59,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-11T02:32:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-11T02:32:59,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-11T02:32:59,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8360 sec 2024-11-11T02:32:59,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.8400 sec 2024-11-11T02:33:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T02:33:00,246 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:00,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:33:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-11T02:33:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T02:33:00,248 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:33:00,248 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:33:00,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:33:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T02:33:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:00,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:33:00,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:33:00,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:00,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:33:00,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:00,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:33:00,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:00,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/10293f25adfe418e9b2f0fcb8a8862f3 is 50, key is test_row_0/A:col10/1731292380361/Put/seqid=0 2024-11-11T02:33:00,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742243_1419 (size=12001) 2024-11-11T02:33:00,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T02:33:00,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:00,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:00,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:00,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:00,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:00,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292380401, max timestamp=1731292380424Num uniques 859; 2024-11-11T02:33:00,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:00,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292380401, max timestamp=1731292380424Num uniques 859; 2024-11-11T02:33:00,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292380401, max timestamp=1731292380424Num uniques 859; 2024-11-11T02:33:00,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:00,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:00,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292440478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:00,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:00,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292440491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:00,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:00,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:00,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292440491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:00,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T02:33:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:00,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T02:33:00,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292440595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:00,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292440605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:00,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292440615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:00,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T02:33:00,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:00,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:00,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:00,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:00,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:00,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/10293f25adfe418e9b2f0fcb8a8862f3 2024-11-11T02:33:00,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/e62766b5e60f4df0ad9cae04e946632b is 50, key is test_row_0/B:col10/1731292380361/Put/seqid=0 2024-11-11T02:33:00,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742244_1420 (size=12001) 2024-11-11T02:33:00,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292440815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:00,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292440825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:00,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292440825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:00,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:00,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:00,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:00,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T02:33:00,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:00,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:00,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:00,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:00,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T02:33:01,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T02:33:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:01,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:01,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292441105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:01,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292441106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:01,118 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in c9be960c11aae9a500eccb669509dc81 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:01,118 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in c9be960c11aae9a500eccb669509dc81 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:01,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292441135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:01,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292441145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:01,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292441145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:01,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:01,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T02:33:01,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:01,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:01,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:01,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:01,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:01,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/e62766b5e60f4df0ad9cae04e946632b 2024-11-11T02:33:01,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/04c91094f53343879906c6920103dc82 is 50, key is test_row_0/C:col10/1731292380361/Put/seqid=0 2024-11-11T02:33:01,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742245_1421 (size=12001) 2024-11-11T02:33:01,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T02:33:01,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:01,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:01,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:01,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:01,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T02:33:01,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T02:33:01,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:01,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:01,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:01,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:01,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:01,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/04c91094f53343879906c6920103dc82 2024-11-11T02:33:01,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/10293f25adfe418e9b2f0fcb8a8862f3 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/10293f25adfe418e9b2f0fcb8a8862f3 2024-11-11T02:33:01,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/10293f25adfe418e9b2f0fcb8a8862f3, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T02:33:01,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/e62766b5e60f4df0ad9cae04e946632b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/e62766b5e60f4df0ad9cae04e946632b 2024-11-11T02:33:01,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/e62766b5e60f4df0ad9cae04e946632b, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T02:33:01,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/04c91094f53343879906c6920103dc82 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/04c91094f53343879906c6920103dc82 2024-11-11T02:33:01,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/04c91094f53343879906c6920103dc82, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T02:33:01,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for c9be960c11aae9a500eccb669509dc81 in 1246ms, sequenceid=117, compaction requested=true 2024-11-11T02:33:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:33:01,608 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:33:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:01,608 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:33:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:33:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:01,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:33:01,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:01,609 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:33:01,609 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/B is initiating minor compaction (all files) 2024-11-11T02:33:01,609 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/B in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:01,609 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/44c1905dde4641a38db38cc9eba29cba, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/03d6a811a9d945a4bbcfd817b3c63a0e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/d241f02a33874a469927c98138b01a7a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/e62766b5e60f4df0ad9cae04e946632b] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=47.0 K 2024-11-11T02:33:01,609 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:33:01,610 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/A is initiating minor compaction (all files) 2024-11-11T02:33:01,610 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/A in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:01,610 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/4d7aad04976044618795025c20520ff7, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/6f991537ddd24b51ba4a0e0b6b32d3e0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/cefbb9c85fa54146a450d9b94772e50f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/10293f25adfe418e9b2f0fcb8a8862f3] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=49.3 K 2024-11-11T02:33:01,610 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 44c1905dde4641a38db38cc9eba29cba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731292376203 2024-11-11T02:33:01,610 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d7aad04976044618795025c20520ff7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731292376203 2024-11-11T02:33:01,610 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 03d6a811a9d945a4bbcfd817b3c63a0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731292376948 2024-11-11T02:33:01,610 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f991537ddd24b51ba4a0e0b6b32d3e0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731292376931 2024-11-11T02:33:01,610 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting d241f02a33874a469927c98138b01a7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731292378145 2024-11-11T02:33:01,610 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting cefbb9c85fa54146a450d9b94772e50f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731292378145 2024-11-11T02:33:01,611 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting e62766b5e60f4df0ad9cae04e946632b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731292380357 2024-11-11T02:33:01,611 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10293f25adfe418e9b2f0fcb8a8862f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731292380357 2024-11-11T02:33:01,617 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#A#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:01,617 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/76062ccd91eb4d38a6f812a4c0ba1813 is 50, key is test_row_0/A:col10/1731292380361/Put/seqid=0 2024-11-11T02:33:01,618 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#B#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:01,618 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/b7c9ecb549a745f0814b7ddefc6c52e3 is 50, key is test_row_0/B:col10/1731292380361/Put/seqid=0 2024-11-11T02:33:01,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742246_1422 (size=12241) 2024-11-11T02:33:01,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T02:33:01,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:01,624 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-11T02:33:01,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:33:01,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:01,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:33:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:33:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:01,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/2cb2c94a7af64a36b46a7dad63ff6133 is 50, key is test_row_0/A:col10/1731292380490/Put/seqid=0 2024-11-11T02:33:01,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742247_1423 (size=12241) 2024-11-11T02:33:01,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742248_1424 (size=12151) 2024-11-11T02:33:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:01,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:01,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292441674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:01,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292441684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:01,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292441685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:01,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:01,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:01,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292441795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:01,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292441805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:01,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292441816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:02,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292442005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:02,024 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/76062ccd91eb4d38a6f812a4c0ba1813 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/76062ccd91eb4d38a6f812a4c0ba1813 2024-11-11T02:33:02,028 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9be960c11aae9a500eccb669509dc81/A of c9be960c11aae9a500eccb669509dc81 into 76062ccd91eb4d38a6f812a4c0ba1813(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:02,028 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:02,028 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/A, priority=12, startTime=1731292381608; duration=0sec 2024-11-11T02:33:02,028 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:02,028 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:A 2024-11-11T02:33:02,028 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:33:02,029 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:33:02,029 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/C is initiating minor compaction (all files) 2024-11-11T02:33:02,029 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/C in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:02,033 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/7bffca6d330a4075bdb66f5c5b7431d5, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/55ff723270e3462b81fdb15890892acc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/22d37c103f544258a12f651860861ba7, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/04c91094f53343879906c6920103dc82] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=47.0 K 2024-11-11T02:33:02,033 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bffca6d330a4075bdb66f5c5b7431d5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731292376203 2024-11-11T02:33:02,034 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55ff723270e3462b81fdb15890892acc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731292376948 2024-11-11T02:33:02,034 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22d37c103f544258a12f651860861ba7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731292378145 2024-11-11T02:33:02,034 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04c91094f53343879906c6920103dc82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731292380357 2024-11-11T02:33:02,035 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/2cb2c94a7af64a36b46a7dad63ff6133 2024-11-11T02:33:02,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292442026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:02,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/8a4486ff1c2249ee91bdd31ffdfc591f is 50, key is test_row_0/B:col10/1731292380490/Put/seqid=0 2024-11-11T02:33:02,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:02,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292442026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:02,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:02,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:02,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:02,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:02,044 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#C#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:02,045 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/9d89cedf61ca420a92f5b76b69c48012 is 50, key is test_row_0/C:col10/1731292380361/Put/seqid=0 2024-11-11T02:33:02,045 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/b7c9ecb549a745f0814b7ddefc6c52e3 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/b7c9ecb549a745f0814b7ddefc6c52e3 2024-11-11T02:33:02,050 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9be960c11aae9a500eccb669509dc81/B of c9be960c11aae9a500eccb669509dc81 into b7c9ecb549a745f0814b7ddefc6c52e3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:02,050 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:02,050 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/B, priority=12, startTime=1731292381608; duration=0sec 2024-11-11T02:33:02,050 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:02,050 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:B 2024-11-11T02:33:02,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742249_1425 (size=12151) 2024-11-11T02:33:02,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742250_1426 (size=12241) 2024-11-11T02:33:02,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292442325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:02,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:02,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:02,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:02,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292442355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:02,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292442355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T02:33:02,455 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/8a4486ff1c2249ee91bdd31ffdfc591f 2024-11-11T02:33:02,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/ab75cd95d13b49869dcb023168a6b3c2 is 50, key is test_row_0/C:col10/1731292380490/Put/seqid=0 2024-11-11T02:33:02,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742251_1427 (size=12151) 2024-11-11T02:33:02,465 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/9d89cedf61ca420a92f5b76b69c48012 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/9d89cedf61ca420a92f5b76b69c48012 2024-11-11T02:33:02,468 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9be960c11aae9a500eccb669509dc81/C of c9be960c11aae9a500eccb669509dc81 into 9d89cedf61ca420a92f5b76b69c48012(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:02,468 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:02,469 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/C, priority=12, startTime=1731292381608; duration=0sec 2024-11-11T02:33:02,469 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:02,469 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:C 2024-11-11T02:33:02,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292442845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:02,865 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/ab75cd95d13b49869dcb023168a6b3c2 2024-11-11T02:33:02,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/2cb2c94a7af64a36b46a7dad63ff6133 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2cb2c94a7af64a36b46a7dad63ff6133 2024-11-11T02:33:02,872 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2cb2c94a7af64a36b46a7dad63ff6133, entries=150, sequenceid=154, filesize=11.9 K 2024-11-11T02:33:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/8a4486ff1c2249ee91bdd31ffdfc591f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/8a4486ff1c2249ee91bdd31ffdfc591f 2024-11-11T02:33:02,876 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/8a4486ff1c2249ee91bdd31ffdfc591f, entries=150, sequenceid=154, filesize=11.9 K 2024-11-11T02:33:02,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/ab75cd95d13b49869dcb023168a6b3c2 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/ab75cd95d13b49869dcb023168a6b3c2 2024-11-11T02:33:02,879 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/ab75cd95d13b49869dcb023168a6b3c2, entries=150, sequenceid=154, filesize=11.9 K 2024-11-11T02:33:02,880 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for c9be960c11aae9a500eccb669509dc81 in 1256ms, sequenceid=154, compaction requested=false 2024-11-11T02:33:02,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:02,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:02,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-11T02:33:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-11T02:33:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:02,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-11T02:33:02,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:33:02,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6330 sec 2024-11-11T02:33:02,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:33:02,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:02,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:33:02,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:02,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:33:02,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:02,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.6370 sec 2024-11-11T02:33:02,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/2211c1bdcd6e4fbeb2276fb8e55bf193 is 50, key is test_row_0/A:col10/1731292382882/Put/seqid=0 2024-11-11T02:33:02,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742252_1428 (size=14541) 2024-11-11T02:33:02,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292382900, max timestamp=1731292382945Num uniques 859; 2024-11-11T02:33:02,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:02,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292382900, max timestamp=1731292382945Num uniques 859; 2024-11-11T02:33:02,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:02,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292382900, max timestamp=1731292382945Num uniques 859; 2024-11-11T02:33:02,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:03,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292443015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:03,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292443022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:03,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292443136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:03,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292443136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:03,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/2211c1bdcd6e4fbeb2276fb8e55bf193 2024-11-11T02:33:03,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/1dd5df47eb24418a8b116563d8536f70 is 50, key is test_row_0/B:col10/1731292382882/Put/seqid=0 2024-11-11T02:33:03,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742253_1429 (size=12151) 2024-11-11T02:33:03,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/1dd5df47eb24418a8b116563d8536f70 2024-11-11T02:33:03,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/6d4b9d2c539e48d9a3613bc5e060cdf6 is 50, key is test_row_0/C:col10/1731292382882/Put/seqid=0 2024-11-11T02:33:03,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742254_1430 (size=12151) 2024-11-11T02:33:03,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292443356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:03,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292443356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:03,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292443675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292443675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:03,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/6d4b9d2c539e48d9a3613bc5e060cdf6 2024-11-11T02:33:03,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/2211c1bdcd6e4fbeb2276fb8e55bf193 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2211c1bdcd6e4fbeb2276fb8e55bf193 2024-11-11T02:33:03,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2211c1bdcd6e4fbeb2276fb8e55bf193, entries=200, sequenceid=167, filesize=14.2 K 2024-11-11T02:33:03,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/1dd5df47eb24418a8b116563d8536f70 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/1dd5df47eb24418a8b116563d8536f70 2024-11-11T02:33:03,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/1dd5df47eb24418a8b116563d8536f70, entries=150, sequenceid=167, filesize=11.9 K 2024-11-11T02:33:03,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/6d4b9d2c539e48d9a3613bc5e060cdf6 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/6d4b9d2c539e48d9a3613bc5e060cdf6 2024-11-11T02:33:03,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/6d4b9d2c539e48d9a3613bc5e060cdf6, entries=150, sequenceid=167, filesize=11.9 K 2024-11-11T02:33:03,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c9be960c11aae9a500eccb669509dc81 in 841ms, sequenceid=167, compaction requested=true 2024-11-11T02:33:03,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:03,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:33:03,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:03,724 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:03,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:33:03,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:03,724 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:03,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:33:03,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:03,725 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:03,725 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:03,725 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/A is initiating minor compaction (all files) 2024-11-11T02:33:03,725 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/B is initiating minor compaction (all files) 2024-11-11T02:33:03,725 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/A in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:03,725 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/B in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:03,725 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/76062ccd91eb4d38a6f812a4c0ba1813, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2cb2c94a7af64a36b46a7dad63ff6133, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2211c1bdcd6e4fbeb2276fb8e55bf193] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=38.0 K 2024-11-11T02:33:03,725 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/b7c9ecb549a745f0814b7ddefc6c52e3, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/8a4486ff1c2249ee91bdd31ffdfc591f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/1dd5df47eb24418a8b116563d8536f70] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=35.7 K 2024-11-11T02:33:03,726 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76062ccd91eb4d38a6f812a4c0ba1813, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731292380357 2024-11-11T02:33:03,726 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting b7c9ecb549a745f0814b7ddefc6c52e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731292380357 2024-11-11T02:33:03,726 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cb2c94a7af64a36b46a7dad63ff6133, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731292380476 2024-11-11T02:33:03,726 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a4486ff1c2249ee91bdd31ffdfc591f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731292380476 2024-11-11T02:33:03,726 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2211c1bdcd6e4fbeb2276fb8e55bf193, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731292381673 2024-11-11T02:33:03,726 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dd5df47eb24418a8b116563d8536f70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731292381673 2024-11-11T02:33:03,731 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#A#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:03,732 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/706c7d1688944f1c9059a99f5ee016e4 is 50, key is test_row_0/A:col10/1731292382882/Put/seqid=0 2024-11-11T02:33:03,732 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#B#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:03,732 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/1a265b1063354aa4bed9e7f10689f83c is 50, key is test_row_0/B:col10/1731292382882/Put/seqid=0 2024-11-11T02:33:03,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742255_1431 (size=12493) 2024-11-11T02:33:03,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742256_1432 (size=12493) 2024-11-11T02:33:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:03,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-11T02:33:03,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:33:03,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:03,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:33:03,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:03,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:33:03,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:03,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/9812e3cb908a446a9860fdf84a161f14 is 50, key is test_row_0/A:col10/1731292383021/Put/seqid=0 2024-11-11T02:33:03,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742257_1433 (size=14541) 2024-11-11T02:33:03,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292443918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:04,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292444035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:04,140 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/706c7d1688944f1c9059a99f5ee016e4 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/706c7d1688944f1c9059a99f5ee016e4 2024-11-11T02:33:04,145 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/A of c9be960c11aae9a500eccb669509dc81 into 706c7d1688944f1c9059a99f5ee016e4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:04,145 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:04,145 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/A, priority=13, startTime=1731292383724; duration=0sec 2024-11-11T02:33:04,145 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:04,145 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:A 2024-11-11T02:33:04,145 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:04,146 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:04,146 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/1a265b1063354aa4bed9e7f10689f83c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/1a265b1063354aa4bed9e7f10689f83c 2024-11-11T02:33:04,146 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/C is initiating minor compaction (all files) 2024-11-11T02:33:04,146 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/C in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:04,146 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/9d89cedf61ca420a92f5b76b69c48012, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/ab75cd95d13b49869dcb023168a6b3c2, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/6d4b9d2c539e48d9a3613bc5e060cdf6] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=35.7 K 2024-11-11T02:33:04,146 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d89cedf61ca420a92f5b76b69c48012, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731292380357 2024-11-11T02:33:04,147 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab75cd95d13b49869dcb023168a6b3c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731292380476 2024-11-11T02:33:04,147 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d4b9d2c539e48d9a3613bc5e060cdf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731292381673 2024-11-11T02:33:04,150 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/B of c9be960c11aae9a500eccb669509dc81 into 1a265b1063354aa4bed9e7f10689f83c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:04,150 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:04,151 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/B, priority=13, startTime=1731292383724; duration=0sec 2024-11-11T02:33:04,151 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:04,151 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:B 2024-11-11T02:33:04,154 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#C#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:04,155 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/0db03edeaa83489796abf84cad058c5b is 50, key is test_row_0/C:col10/1731292382882/Put/seqid=0 2024-11-11T02:33:04,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742258_1434 (size=12493) 2024-11-11T02:33:04,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292444185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:04,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292444185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:04,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292444255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:04,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:04,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:04,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:04,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/9812e3cb908a446a9860fdf84a161f14 2024-11-11T02:33:04,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/901eca06a1c54878b62c97d73a2349e6 is 50, key is test_row_0/B:col10/1731292383021/Put/seqid=0 2024-11-11T02:33:04,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742259_1435 (size=12151) 2024-11-11T02:33:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T02:33:04,426 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:04,426 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:33:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-11T02:33:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T02:33:04,428 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:33:04,428 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:33:04,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:33:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T02:33:04,562 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/0db03edeaa83489796abf84cad058c5b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0db03edeaa83489796abf84cad058c5b 2024-11-11T02:33:04,565 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/C of c9be960c11aae9a500eccb669509dc81 into 0db03edeaa83489796abf84cad058c5b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:04,565 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:04,565 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/C, priority=13, startTime=1731292383724; duration=0sec 2024-11-11T02:33:04,565 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:04,565 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:C 2024-11-11T02:33:04,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T02:33:04,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:04,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:04,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:04,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:04,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:04,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:04,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292444576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:04,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:04,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:04,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:04,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/901eca06a1c54878b62c97d73a2349e6 2024-11-11T02:33:04,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/f05d68a718d9465c8729065823ebeb5b is 50, key is test_row_0/C:col10/1731292383021/Put/seqid=0 2024-11-11T02:33:04,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742260_1436 (size=12151) 2024-11-11T02:33:04,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/f05d68a718d9465c8729065823ebeb5b 2024-11-11T02:33:04,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/9812e3cb908a446a9860fdf84a161f14 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/9812e3cb908a446a9860fdf84a161f14 2024-11-11T02:33:04,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/9812e3cb908a446a9860fdf84a161f14, entries=200, sequenceid=204, filesize=14.2 K 2024-11-11T02:33:04,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/901eca06a1c54878b62c97d73a2349e6 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/901eca06a1c54878b62c97d73a2349e6 2024-11-11T02:33:04,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/901eca06a1c54878b62c97d73a2349e6, entries=150, sequenceid=204, filesize=11.9 K 2024-11-11T02:33:04,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/f05d68a718d9465c8729065823ebeb5b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/f05d68a718d9465c8729065823ebeb5b 2024-11-11T02:33:04,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/f05d68a718d9465c8729065823ebeb5b, entries=150, sequenceid=204, filesize=11.9 K 2024-11-11T02:33:04,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for c9be960c11aae9a500eccb669509dc81 in 857ms, sequenceid=204, compaction requested=false 2024-11-11T02:33:04,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:04,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T02:33:04,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:04,733 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T02:33:04,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:33:04,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:04,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:33:04,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:04,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:33:04,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:04,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/bf0f35cb34cf419b9c98ed33f92fab55 is 50, key is test_row_1/A:col10/1731292383917/Put/seqid=0 2024-11-11T02:33:04,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742261_1437 (size=9757) 2024-11-11T02:33:04,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T02:33:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T02:33:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:05,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:05,145 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/bf0f35cb34cf419b9c98ed33f92fab55 2024-11-11T02:33:05,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/8671361ca5a24dbd98849746366d6729 is 50, key is test_row_1/B:col10/1731292383917/Put/seqid=0 2024-11-11T02:33:05,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742262_1438 (size=9757) 2024-11-11T02:33:05,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292385150, max timestamp=1731292385167Num uniques 859; 2024-11-11T02:33:05,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292385150, max timestamp=1731292385167Num uniques 859; 2024-11-11T02:33:05,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:05,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:05,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292385150, max timestamp=1731292385167Num uniques 859; 2024-11-11T02:33:05,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:05,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292445207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:05,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292445211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:05,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292445212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:05,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:05,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292445216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:05,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:05,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292445216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:05,228 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in c9be960c11aae9a500eccb669509dc81 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:05,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:05,229 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in c9be960c11aae9a500eccb669509dc81 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:05,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:05,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292445325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:05,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292445325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:05,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292445335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:05,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:05,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292445546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:05,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292445546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:05,554 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/8671361ca5a24dbd98849746366d6729 2024-11-11T02:33:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:05,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/a56f4f9fb6214e5caa0397291514bab4 is 50, key is test_row_1/C:col10/1731292383917/Put/seqid=0 2024-11-11T02:33:05,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742263_1439 (size=9757) 2024-11-11T02:33:05,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292445555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T02:33:05,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292445865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:05,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292445865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:05,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292445876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:05,963 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/a56f4f9fb6214e5caa0397291514bab4 2024-11-11T02:33:05,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/bf0f35cb34cf419b9c98ed33f92fab55 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/bf0f35cb34cf419b9c98ed33f92fab55 2024-11-11T02:33:05,969 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/bf0f35cb34cf419b9c98ed33f92fab55, entries=100, sequenceid=216, filesize=9.5 K 2024-11-11T02:33:05,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/8671361ca5a24dbd98849746366d6729 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/8671361ca5a24dbd98849746366d6729 2024-11-11T02:33:05,978 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/8671361ca5a24dbd98849746366d6729, entries=100, sequenceid=216, filesize=9.5 K 2024-11-11T02:33:05,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/a56f4f9fb6214e5caa0397291514bab4 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/a56f4f9fb6214e5caa0397291514bab4 2024-11-11T02:33:05,982 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/a56f4f9fb6214e5caa0397291514bab4, entries=100, sequenceid=216, filesize=9.5 K 2024-11-11T02:33:05,983 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for c9be960c11aae9a500eccb669509dc81 in 1250ms, sequenceid=216, compaction requested=true 2024-11-11T02:33:05,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:05,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:05,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-11T02:33:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-11T02:33:05,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-11T02:33:05,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5560 sec 2024-11-11T02:33:05,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.5600 sec 2024-11-11T02:33:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:06,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-11T02:33:06,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:33:06,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:06,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:33:06,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:06,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:33:06,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:06,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/36340238faf241fca697fca6c9cf715e is 50, key is test_row_0/A:col10/1731292386382/Put/seqid=0 2024-11-11T02:33:06,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742264_1440 (size=14541) 2024-11-11T02:33:06,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292446403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:06,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292446404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292446407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:06,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:06,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292446515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:06,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:06,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292446525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:06,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292446525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T02:33:06,576 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:06,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:33:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-11T02:33:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T02:33:06,578 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:33:06,578 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:33:06,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:33:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T02:33:06,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T02:33:06,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:06,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:06,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292446735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:06,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292446745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:06,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:06,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292446745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:06,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/36340238faf241fca697fca6c9cf715e 2024-11-11T02:33:06,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/9622d0d1b4cd46c49697f7163c130ee6 is 50, key is test_row_0/B:col10/1731292386382/Put/seqid=0 2024-11-11T02:33:06,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742265_1441 (size=12151) 2024-11-11T02:33:06,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T02:33:06,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:06,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:06,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:06,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:06,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:06,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T02:33:07,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T02:33:07,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:07,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:07,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:07,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:07,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:07,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292447056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:07,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:07,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292447065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:07,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292447065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:07,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T02:33:07,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:07,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:07,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:07,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:07,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:07,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/9622d0d1b4cd46c49697f7163c130ee6 2024-11-11T02:33:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T02:33:07,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/18eb0b04a02749ea86e8e9d67628fac0 is 50, key is test_row_0/C:col10/1731292386382/Put/seqid=0 2024-11-11T02:33:07,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742266_1442 (size=12151) 2024-11-11T02:33:07,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292447236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:07,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292447236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:07,242 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in c9be960c11aae9a500eccb669509dc81 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:07,242 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in c9be960c11aae9a500eccb669509dc81 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:07,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:07,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:07,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T02:33:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:07,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:07,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T02:33:07,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:07,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:07,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:07,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:07,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:07,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292447565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:07,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:07,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292447585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:07,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292447586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:07,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/18eb0b04a02749ea86e8e9d67628fac0 2024-11-11T02:33:07,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/36340238faf241fca697fca6c9cf715e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/36340238faf241fca697fca6c9cf715e 2024-11-11T02:33:07,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/36340238faf241fca697fca6c9cf715e, entries=200, sequenceid=254, filesize=14.2 K 2024-11-11T02:33:07,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/9622d0d1b4cd46c49697f7163c130ee6 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/9622d0d1b4cd46c49697f7163c130ee6 2024-11-11T02:33:07,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/9622d0d1b4cd46c49697f7163c130ee6, entries=150, sequenceid=254, filesize=11.9 K 2024-11-11T02:33:07,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/18eb0b04a02749ea86e8e9d67628fac0 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/18eb0b04a02749ea86e8e9d67628fac0 2024-11-11T02:33:07,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/18eb0b04a02749ea86e8e9d67628fac0, entries=150, sequenceid=254, filesize=11.9 K 2024-11-11T02:33:07,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for c9be960c11aae9a500eccb669509dc81 in 1243ms, sequenceid=254, compaction requested=true 2024-11-11T02:33:07,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:07,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:33:07,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:07,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:33:07,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:07,627 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:33:07,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:33:07,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:07,627 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:33:07,628 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:33:07,628 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51332 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:33:07,628 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/B is initiating minor compaction (all files) 2024-11-11T02:33:07,628 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/A is initiating minor compaction (all files) 2024-11-11T02:33:07,628 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/A in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:07,628 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/B in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:07,628 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/1a265b1063354aa4bed9e7f10689f83c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/901eca06a1c54878b62c97d73a2349e6, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/8671361ca5a24dbd98849746366d6729, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/9622d0d1b4cd46c49697f7163c130ee6] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=45.5 K 2024-11-11T02:33:07,628 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/706c7d1688944f1c9059a99f5ee016e4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/9812e3cb908a446a9860fdf84a161f14, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/bf0f35cb34cf419b9c98ed33f92fab55, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/36340238faf241fca697fca6c9cf715e] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=50.1 K 2024-11-11T02:33:07,629 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a265b1063354aa4bed9e7f10689f83c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731292381673 2024-11-11T02:33:07,629 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 706c7d1688944f1c9059a99f5ee016e4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731292381673 2024-11-11T02:33:07,629 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9812e3cb908a446a9860fdf84a161f14, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292382978 2024-11-11T02:33:07,629 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 901eca06a1c54878b62c97d73a2349e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292383013 2024-11-11T02:33:07,629 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf0f35cb34cf419b9c98ed33f92fab55, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731292383889 2024-11-11T02:33:07,629 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 8671361ca5a24dbd98849746366d6729, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731292383889 2024-11-11T02:33:07,629 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36340238faf241fca697fca6c9cf715e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731292385196 2024-11-11T02:33:07,629 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 9622d0d1b4cd46c49697f7163c130ee6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731292385196 2024-11-11T02:33:07,637 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#A#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:07,638 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/7ba2f45ad5fa406d87198da78b2bf154 is 50, key is test_row_0/A:col10/1731292386382/Put/seqid=0 2024-11-11T02:33:07,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T02:33:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:07,646 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T02:33:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:33:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:33:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:07,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:33:07,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:07,647 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#B#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:07,648 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/cade0795171e48babc15f99cf47b4a2b is 50, key is test_row_0/B:col10/1731292386382/Put/seqid=0 2024-11-11T02:33:07,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/c098f14562ca459a8d25af145f229940 is 50, key is test_row_0/A:col10/1731292386403/Put/seqid=0 2024-11-11T02:33:07,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742267_1443 (size=12629) 2024-11-11T02:33:07,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742268_1444 (size=12629) 2024-11-11T02:33:07,663 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/7ba2f45ad5fa406d87198da78b2bf154 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/7ba2f45ad5fa406d87198da78b2bf154 2024-11-11T02:33:07,664 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/cade0795171e48babc15f99cf47b4a2b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/cade0795171e48babc15f99cf47b4a2b 2024-11-11T02:33:07,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742269_1445 (size=12301) 2024-11-11T02:33:07,670 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9be960c11aae9a500eccb669509dc81/A of c9be960c11aae9a500eccb669509dc81 into 7ba2f45ad5fa406d87198da78b2bf154(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:07,670 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:07,670 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/A, priority=12, startTime=1731292387627; duration=0sec 2024-11-11T02:33:07,670 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:07,670 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:A 2024-11-11T02:33:07,670 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:33:07,671 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9be960c11aae9a500eccb669509dc81/B of c9be960c11aae9a500eccb669509dc81 into cade0795171e48babc15f99cf47b4a2b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:07,671 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:07,671 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/B, priority=12, startTime=1731292387627; duration=0sec 2024-11-11T02:33:07,671 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:07,671 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:B 2024-11-11T02:33:07,671 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:33:07,671 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/C is initiating minor compaction (all files) 2024-11-11T02:33:07,671 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/C in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:07,671 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0db03edeaa83489796abf84cad058c5b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/f05d68a718d9465c8729065823ebeb5b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/a56f4f9fb6214e5caa0397291514bab4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/18eb0b04a02749ea86e8e9d67628fac0] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=45.5 K 2024-11-11T02:33:07,672 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0db03edeaa83489796abf84cad058c5b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731292381673 2024-11-11T02:33:07,672 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting f05d68a718d9465c8729065823ebeb5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731292383013 2024-11-11T02:33:07,672 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56f4f9fb6214e5caa0397291514bab4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731292383889 2024-11-11T02:33:07,673 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18eb0b04a02749ea86e8e9d67628fac0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731292385196 2024-11-11T02:33:07,679 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#C#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:07,679 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/be5cfbbdd1234e2380de599465f3bbdb is 50, key is test_row_0/C:col10/1731292386382/Put/seqid=0 2024-11-11T02:33:07,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742270_1446 (size=12629) 2024-11-11T02:33:07,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T02:33:08,068 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/c098f14562ca459a8d25af145f229940 2024-11-11T02:33:08,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/fa47440cd4ff4e85a89f553277648e75 is 50, key is test_row_0/B:col10/1731292386403/Put/seqid=0 2024-11-11T02:33:08,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742271_1447 (size=12301) 2024-11-11T02:33:08,087 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/be5cfbbdd1234e2380de599465f3bbdb as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/be5cfbbdd1234e2380de599465f3bbdb 2024-11-11T02:33:08,093 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9be960c11aae9a500eccb669509dc81/C of c9be960c11aae9a500eccb669509dc81 into be5cfbbdd1234e2380de599465f3bbdb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:08,093 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:08,093 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/C, priority=12, startTime=1731292387627; duration=0sec 2024-11-11T02:33:08,093 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:08,093 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:C 2024-11-11T02:33:08,484 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/fa47440cd4ff4e85a89f553277648e75 2024-11-11T02:33:08,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/80d46911e693483d94691364c9ea38d8 is 50, key is test_row_0/C:col10/1731292386403/Put/seqid=0 2024-11-11T02:33:08,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742272_1448 (size=12301) 2024-11-11T02:33:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:08,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:08,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292388623, max timestamp=1731292388642Num uniques 859; 2024-11-11T02:33:08,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:08,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292388623, max timestamp=1731292388642Num uniques 859; 2024-11-11T02:33:08,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292388623, max timestamp=1731292388642Num uniques 859; 2024-11-11T02:33:08,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:08,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:08,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292448699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:08,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292448701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:08,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731292388691, max timestamp=1731292388709Num uniques 1068; 2024-11-11T02:33:08,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T02:33:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:08,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731292388691, max timestamp=1731292388709Num uniques 1068; 2024-11-11T02:33:08,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T02:33:08,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731292388691, max timestamp=1731292388709Num uniques 1068; 2024-11-11T02:33:08,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T02:33:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T02:33:08,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731292388796, max timestamp=1731292388823Num uniques 1077; 2024-11-11T02:33:08,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-11T02:33:08,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731292388796, max timestamp=1731292388823Num uniques 1077; 2024-11-11T02:33:08,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731292388796, max timestamp=1731292388823Num uniques 1077; 2024-11-11T02:33:08,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-11T02:33:08,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-11T02:33:08,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292448868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:08,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292448876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:08,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292448880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:08,902 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/80d46911e693483d94691364c9ea38d8 2024-11-11T02:33:08,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/c098f14562ca459a8d25af145f229940 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/c098f14562ca459a8d25af145f229940 2024-11-11T02:33:08,909 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/c098f14562ca459a8d25af145f229940, entries=150, sequenceid=262, filesize=12.0 K 2024-11-11T02:33:08,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/fa47440cd4ff4e85a89f553277648e75 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/fa47440cd4ff4e85a89f553277648e75 2024-11-11T02:33:08,912 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/fa47440cd4ff4e85a89f553277648e75, entries=150, sequenceid=262, filesize=12.0 K 2024-11-11T02:33:08,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/80d46911e693483d94691364c9ea38d8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/80d46911e693483d94691364c9ea38d8 2024-11-11T02:33:08,915 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/80d46911e693483d94691364c9ea38d8, entries=150, sequenceid=262, filesize=12.0 K 2024-11-11T02:33:08,916 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for c9be960c11aae9a500eccb669509dc81 in 1270ms, sequenceid=262, compaction requested=false 2024-11-11T02:33:08,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:08,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:08,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-11T02:33:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-11T02:33:08,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-11T02:33:08,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3390 sec 2024-11-11T02:33:08,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.3430 sec 2024-11-11T02:33:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:08,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=154.31 KB heapSize=434.39 KB 2024-11-11T02:33:08,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:33:08,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:08,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:33:08,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:08,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:33:08,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:08,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/f789008954cc4bc4941055a2d73b8b2d is 50, key is test_row_0/A:col10/1731292388854/Put/seqid=0 2024-11-11T02:33:09,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742273_1449 (size=12301) 2024-11-11T02:33:09,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292449006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:09,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292449006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:09,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292449007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:09,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292449115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:09,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292449115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:09,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:09,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:09,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292449125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:09,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292449336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:09,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292449336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:09,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292449336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:09,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/f789008954cc4bc4941055a2d73b8b2d 2024-11-11T02:33:09,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/d4cf4374035c4a9eb5c4356a1713007f is 50, key is test_row_0/B:col10/1731292388854/Put/seqid=0 2024-11-11T02:33:09,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742274_1450 (size=12301) 2024-11-11T02:33:09,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292449646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:09,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:09,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292449646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:09,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:09,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:09,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292449646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:09,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:09,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/d4cf4374035c4a9eb5c4356a1713007f 2024-11-11T02:33:09,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/330a081c65084d9d939216f674d22814 is 50, key is test_row_0/C:col10/1731292388854/Put/seqid=0 2024-11-11T02:33:09,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742275_1451 (size=12301) 2024-11-11T02:33:09,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/330a081c65084d9d939216f674d22814 2024-11-11T02:33:09,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/f789008954cc4bc4941055a2d73b8b2d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/f789008954cc4bc4941055a2d73b8b2d 2024-11-11T02:33:09,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/f789008954cc4bc4941055a2d73b8b2d, entries=150, sequenceid=337, filesize=12.0 K 2024-11-11T02:33:09,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/d4cf4374035c4a9eb5c4356a1713007f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/d4cf4374035c4a9eb5c4356a1713007f 2024-11-11T02:33:09,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/d4cf4374035c4a9eb5c4356a1713007f, entries=150, sequenceid=337, filesize=12.0 K 2024-11-11T02:33:09,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/330a081c65084d9d939216f674d22814 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/330a081c65084d9d939216f674d22814 2024-11-11T02:33:09,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/330a081c65084d9d939216f674d22814, entries=150, sequenceid=337, filesize=12.0 K 2024-11-11T02:33:09,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~434.34 KB/444768, currentSize=26.84 KB/27480 for c9be960c11aae9a500eccb669509dc81 in 871ms, sequenceid=337, compaction requested=true 2024-11-11T02:33:09,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:09,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:33:09,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:09,864 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:09,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:33:09,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:09,864 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:09,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:33:09,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:09,865 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:09,865 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:09,865 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/A is initiating minor compaction (all files) 2024-11-11T02:33:09,865 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/B is initiating minor compaction (all files) 2024-11-11T02:33:09,865 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/A in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:09,865 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/B in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:09,865 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/7ba2f45ad5fa406d87198da78b2bf154, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/c098f14562ca459a8d25af145f229940, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/f789008954cc4bc4941055a2d73b8b2d] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=36.4 K 2024-11-11T02:33:09,865 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/cade0795171e48babc15f99cf47b4a2b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/fa47440cd4ff4e85a89f553277648e75, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/d4cf4374035c4a9eb5c4356a1713007f] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=36.4 K 2024-11-11T02:33:09,865 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ba2f45ad5fa406d87198da78b2bf154, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731292385196 2024-11-11T02:33:09,865 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting cade0795171e48babc15f99cf47b4a2b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731292385196 2024-11-11T02:33:09,866 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting c098f14562ca459a8d25af145f229940, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731292386402 2024-11-11T02:33:09,866 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting fa47440cd4ff4e85a89f553277648e75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731292386402 2024-11-11T02:33:09,866 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting f789008954cc4bc4941055a2d73b8b2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731292388854 2024-11-11T02:33:09,866 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting d4cf4374035c4a9eb5c4356a1713007f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731292388854 2024-11-11T02:33:09,878 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#A#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:09,878 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/a71433d64822466ea75efba25d0545ee is 50, key is test_row_0/A:col10/1731292388854/Put/seqid=0 2024-11-11T02:33:09,882 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#B#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:09,882 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/82294d8899944584aa42289cb684571f is 50, key is test_row_0/B:col10/1731292388854/Put/seqid=0 2024-11-11T02:33:09,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742276_1452 (size=12881) 2024-11-11T02:33:09,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742277_1453 (size=12881) 2024-11-11T02:33:09,902 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/a71433d64822466ea75efba25d0545ee as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/a71433d64822466ea75efba25d0545ee 2024-11-11T02:33:09,914 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/A of c9be960c11aae9a500eccb669509dc81 into a71433d64822466ea75efba25d0545ee(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:09,914 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:09,914 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/A, priority=13, startTime=1731292389864; duration=0sec 2024-11-11T02:33:09,914 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:09,914 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:A 2024-11-11T02:33:09,914 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:09,915 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:09,915 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/C is initiating minor compaction (all files) 2024-11-11T02:33:09,915 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/C in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:09,915 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/be5cfbbdd1234e2380de599465f3bbdb, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/80d46911e693483d94691364c9ea38d8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/330a081c65084d9d939216f674d22814] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=36.4 K 2024-11-11T02:33:09,916 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting be5cfbbdd1234e2380de599465f3bbdb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731292385196 2024-11-11T02:33:09,916 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80d46911e693483d94691364c9ea38d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731292386402 2024-11-11T02:33:09,916 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 330a081c65084d9d939216f674d22814, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731292388854 2024-11-11T02:33:09,925 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:09,926 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/0bfab1709cc6466b83a51544af16e2d3 is 50, key is test_row_0/C:col10/1731292388854/Put/seqid=0 2024-11-11T02:33:09,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742278_1454 (size=12881) 2024-11-11T02:33:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:10,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:33:10,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:33:10,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:10,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:33:10,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:10,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:33:10,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:10,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/2246cb5ad72d4d6da727f2e4055c402b is 50, key is test_row_0/A:col10/1731292390170/Put/seqid=0 2024-11-11T02:33:10,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742279_1455 (size=17177) 2024-11-11T02:33:10,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292390198, max timestamp=1731292390221Num uniques 859; 2024-11-11T02:33:10,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:33:10,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292390198, max timestamp=1731292390221Num uniques 859; 2024-11-11T02:33:10,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731292390198, max timestamp=1731292390221Num uniques 859; 2024-11-11T02:33:10,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:33:10,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T02:33:10,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292450281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:10,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292450283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:10,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:10,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292450284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:10,303 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/82294d8899944584aa42289cb684571f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/82294d8899944584aa42289cb684571f 2024-11-11T02:33:10,306 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/B of c9be960c11aae9a500eccb669509dc81 into 82294d8899944584aa42289cb684571f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:10,306 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:10,306 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/B, priority=13, startTime=1731292389864; duration=0sec 2024-11-11T02:33:10,306 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:10,306 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:B 2024-11-11T02:33:10,339 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/0bfab1709cc6466b83a51544af16e2d3 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0bfab1709cc6466b83a51544af16e2d3 2024-11-11T02:33:10,343 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/C of c9be960c11aae9a500eccb669509dc81 into 0bfab1709cc6466b83a51544af16e2d3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:10,343 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:10,343 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/C, priority=13, startTime=1731292389864; duration=0sec 2024-11-11T02:33:10,343 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:10,343 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:C 2024-11-11T02:33:10,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292450395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:10,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292450405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292450406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:10,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/2246cb5ad72d4d6da727f2e4055c402b 2024-11-11T02:33:10,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/2009f570ea8a477b8d5b50d556e1fb65 is 50, key is test_row_0/B:col10/1731292390170/Put/seqid=0 2024-11-11T02:33:10,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742280_1456 (size=9857) 2024-11-11T02:33:10,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292450615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:10,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292450625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292450625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:10,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T02:33:10,736 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:10,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:33:10,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-11T02:33:10,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T02:33:10,737 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:33:10,738 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:33:10,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:33:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T02:33:10,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T02:33:10,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:10,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:10,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:10,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:10,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:10,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292450935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:10,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292450945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:10,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292450945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:10,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:10,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:10,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/2009f570ea8a477b8d5b50d556e1fb65 2024-11-11T02:33:10,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/0ac791b103f6411685e0fd9c1802a845 is 50, key is test_row_0/C:col10/1731292390170/Put/seqid=0 2024-11-11T02:33:11,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742281_1457 (size=9857) 2024-11-11T02:33:11,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T02:33:11,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:11,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:11,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:11,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T02:33:11,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T02:33:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:11,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:11,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:11,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34136 deadline: 1731292451256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:11,262 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in c9be960c11aae9a500eccb669509dc81 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:11,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:11,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34118 deadline: 1731292451285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:11,290 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in c9be960c11aae9a500eccb669509dc81 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:11,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T02:33:11,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:11,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:11,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:11,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T02:33:11,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/0ac791b103f6411685e0fd9c1802a845 2024-11-11T02:33:11,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/2246cb5ad72d4d6da727f2e4055c402b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2246cb5ad72d4d6da727f2e4055c402b 2024-11-11T02:33:11,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2246cb5ad72d4d6da727f2e4055c402b, entries=250, sequenceid=348, filesize=16.8 K 2024-11-11T02:33:11,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/2009f570ea8a477b8d5b50d556e1fb65 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/2009f570ea8a477b8d5b50d556e1fb65 2024-11-11T02:33:11,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/2009f570ea8a477b8d5b50d556e1fb65, entries=100, sequenceid=348, filesize=9.6 K 2024-11-11T02:33:11,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/0ac791b103f6411685e0fd9c1802a845 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0ac791b103f6411685e0fd9c1802a845 2024-11-11T02:33:11,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0ac791b103f6411685e0fd9c1802a845, entries=100, sequenceid=348, filesize=9.6 K 2024-11-11T02:33:11,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for c9be960c11aae9a500eccb669509dc81 in 1245ms, sequenceid=348, compaction requested=false 2024-11-11T02:33:11,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:11,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-11T02:33:11,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:33:11,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:11,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:33:11,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:11,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:33:11,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:11,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/6e83ef2906d34e9ca2ae78d98add9968 is 50, key is test_row_0/A:col10/1731292390281/Put/seqid=0 2024-11-11T02:33:11,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742282_1458 (size=14741) 2024-11-11T02:33:11,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/6e83ef2906d34e9ca2ae78d98add9968 2024-11-11T02:33:11,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/7aeab7f73aee485c91b88d375f269eef is 50, key is test_row_0/B:col10/1731292390281/Put/seqid=0 2024-11-11T02:33:11,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742283_1459 (size=12301) 2024-11-11T02:33:11,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292451483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:11,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:11,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292451488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:11,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:11,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292451496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:11,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T02:33:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:11,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:11,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:11,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:11,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292451605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:11,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292451605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:11,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292451605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:11,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T02:33:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:11,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T02:33:11,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:11,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:11,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:11,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292451815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:11,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292451825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:11,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:11,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292451825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:11,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:11,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T02:33:11,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/7aeab7f73aee485c91b88d375f269eef 2024-11-11T02:33:11,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/d2963b5aa02c4e1cae2b86a272a4cb31 is 50, key is test_row_0/C:col10/1731292390281/Put/seqid=0 2024-11-11T02:33:11,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742284_1460 (size=12301) 2024-11-11T02:33:11,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T02:33:11,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:11,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:11,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:11,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:12,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T02:33:12,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:12,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:12,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:12,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:12,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:12,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292452125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:12,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292452146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292452146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:12,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T02:33:12,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:12,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:12,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:12,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:12,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:12,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/d2963b5aa02c4e1cae2b86a272a4cb31 2024-11-11T02:33:12,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/6e83ef2906d34e9ca2ae78d98add9968 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/6e83ef2906d34e9ca2ae78d98add9968 2024-11-11T02:33:12,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/6e83ef2906d34e9ca2ae78d98add9968, entries=200, sequenceid=386, filesize=14.4 K 2024-11-11T02:33:12,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/7aeab7f73aee485c91b88d375f269eef as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/7aeab7f73aee485c91b88d375f269eef 2024-11-11T02:33:12,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/7aeab7f73aee485c91b88d375f269eef, entries=150, sequenceid=386, filesize=12.0 K 2024-11-11T02:33:12,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/d2963b5aa02c4e1cae2b86a272a4cb31 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/d2963b5aa02c4e1cae2b86a272a4cb31 2024-11-11T02:33:12,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/d2963b5aa02c4e1cae2b86a272a4cb31, entries=150, sequenceid=386, filesize=12.0 K 2024-11-11T02:33:12,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for c9be960c11aae9a500eccb669509dc81 in 843ms, sequenceid=386, compaction requested=true 2024-11-11T02:33:12,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:12,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:33:12,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:12,302 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:12,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:33:12,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:12,302 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:12,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:33:12,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:12,303 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:12,303 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44799 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:12,303 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/B is initiating minor compaction (all files) 2024-11-11T02:33:12,303 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/A is initiating minor compaction (all files) 2024-11-11T02:33:12,303 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/B in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:12,303 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/A in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:12,303 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/82294d8899944584aa42289cb684571f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/2009f570ea8a477b8d5b50d556e1fb65, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/7aeab7f73aee485c91b88d375f269eef] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=34.2 K 2024-11-11T02:33:12,303 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/a71433d64822466ea75efba25d0545ee, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2246cb5ad72d4d6da727f2e4055c402b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/6e83ef2906d34e9ca2ae78d98add9968] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=43.7 K 2024-11-11T02:33:12,303 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting a71433d64822466ea75efba25d0545ee, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731292388854 2024-11-11T02:33:12,303 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 82294d8899944584aa42289cb684571f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731292388854 2024-11-11T02:33:12,303 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2246cb5ad72d4d6da727f2e4055c402b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731292389004 2024-11-11T02:33:12,303 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 2009f570ea8a477b8d5b50d556e1fb65, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731292390170 2024-11-11T02:33:12,303 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aeab7f73aee485c91b88d375f269eef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731292390269 2024-11-11T02:33:12,303 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e83ef2906d34e9ca2ae78d98add9968, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731292390269 2024-11-11T02:33:12,310 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:12,310 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/cb3c41622dcb4b3187fc497ac58b2cae is 50, key is test_row_0/A:col10/1731292390281/Put/seqid=0 2024-11-11T02:33:12,313 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#B#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:12,313 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/61af7b6ba736464e8d587ac8a56502bd is 50, key is test_row_0/B:col10/1731292390281/Put/seqid=0 2024-11-11T02:33:12,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742285_1461 (size=12983) 2024-11-11T02:33:12,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742286_1462 (size=12983) 2024-11-11T02:33:12,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T02:33:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:12,417 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T02:33:12,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:33:12,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:12,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:33:12,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:12,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:33:12,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:12,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/d2a11984b5e34fc39eaa60970bb1bb63 is 50, key is test_row_0/A:col10/1731292391471/Put/seqid=0 2024-11-11T02:33:12,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742287_1463 (size=12301) 2024-11-11T02:33:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:12,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. as already flushing 2024-11-11T02:33:12,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731292392684, max timestamp=1731292392702Num uniques 859; 2024-11-11T02:33:12,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T02:33:12,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731292392684, max timestamp=1731292392702Num uniques 859; 2024-11-11T02:33:12,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731292392684, max timestamp=1731292392702Num uniques 859; 2024-11-11T02:33:12,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T02:33:12,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T02:33:12,724 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/cb3c41622dcb4b3187fc497ac58b2cae as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/cb3c41622dcb4b3187fc497ac58b2cae 2024-11-11T02:33:12,725 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/61af7b6ba736464e8d587ac8a56502bd as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/61af7b6ba736464e8d587ac8a56502bd 2024-11-11T02:33:12,727 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/A of c9be960c11aae9a500eccb669509dc81 into cb3c41622dcb4b3187fc497ac58b2cae(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:12,728 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:12,728 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/A, priority=13, startTime=1731292392302; duration=0sec 2024-11-11T02:33:12,728 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:12,728 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:A 2024-11-11T02:33:12,728 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:12,728 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/B of c9be960c11aae9a500eccb669509dc81 into 61af7b6ba736464e8d587ac8a56502bd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:12,728 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:12,729 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:12,729 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/B, priority=13, startTime=1731292392302; duration=0sec 2024-11-11T02:33:12,729 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:12,729 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/C is initiating minor compaction (all files) 2024-11-11T02:33:12,729 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:B 2024-11-11T02:33:12,729 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/C in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:12,729 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0bfab1709cc6466b83a51544af16e2d3, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0ac791b103f6411685e0fd9c1802a845, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/d2963b5aa02c4e1cae2b86a272a4cb31] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=34.2 K 2024-11-11T02:33:12,729 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bfab1709cc6466b83a51544af16e2d3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731292388854 2024-11-11T02:33:12,729 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ac791b103f6411685e0fd9c1802a845, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731292390170 2024-11-11T02:33:12,730 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2963b5aa02c4e1cae2b86a272a4cb31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731292390269 2024-11-11T02:33:12,734 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#C#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:12,735 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/2a2670a013324fa2b0d7d797430c1dd9 is 50, key is test_row_0/C:col10/1731292390281/Put/seqid=0 2024-11-11T02:33:12,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742288_1464 (size=12983) 2024-11-11T02:33:12,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292452756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:12,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292452761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:12,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292452765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:12,828 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/d2a11984b5e34fc39eaa60970bb1bb63 2024-11-11T02:33:12,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/e93b9b70541849328e1abad209ada791 is 50, key is test_row_0/B:col10/1731292391471/Put/seqid=0 2024-11-11T02:33:12,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742289_1465 (size=12301) 2024-11-11T02:33:12,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292452875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:12,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:12,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292452875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:12,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:12,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292452875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:12,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:12,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T02:33:13,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:13,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292453085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:13,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:13,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:13,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:13,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292453085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:13,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:13,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:13,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:13,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292453085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:13,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:13,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:13,142 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/2a2670a013324fa2b0d7d797430c1dd9 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/2a2670a013324fa2b0d7d797430c1dd9 2024-11-11T02:33:13,146 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/C of c9be960c11aae9a500eccb669509dc81 into 2a2670a013324fa2b0d7d797430c1dd9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:13,146 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:13,146 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/C, priority=13, startTime=1731292392302; duration=0sec 2024-11-11T02:33:13,146 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:13,146 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:C 2024-11-11T02:33:13,237 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/e93b9b70541849328e1abad209ada791 2024-11-11T02:33:13,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/4e0dcbfb87684849a5c43f385d6d78ef is 50, key is test_row_0/C:col10/1731292391471/Put/seqid=0 2024-11-11T02:33:13,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742290_1466 (size=12301) 2024-11-11T02:33:13,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1731292453406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:13,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34094 deadline: 1731292453406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34090 deadline: 1731292453406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 , the old value is region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., hostname=370bc2ade342,38037,1731292247443, seqNum=2 because the exception is null or not the one we care about 2024-11-11T02:33:13,646 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/4e0dcbfb87684849a5c43f385d6d78ef 2024-11-11T02:33:13,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/d2a11984b5e34fc39eaa60970bb1bb63 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/d2a11984b5e34fc39eaa60970bb1bb63 2024-11-11T02:33:13,653 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/d2a11984b5e34fc39eaa60970bb1bb63, entries=150, sequenceid=395, filesize=12.0 K 2024-11-11T02:33:13,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/e93b9b70541849328e1abad209ada791 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/e93b9b70541849328e1abad209ada791 2024-11-11T02:33:13,656 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/e93b9b70541849328e1abad209ada791, entries=150, sequenceid=395, filesize=12.0 K 2024-11-11T02:33:13,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/4e0dcbfb87684849a5c43f385d6d78ef as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/4e0dcbfb87684849a5c43f385d6d78ef 2024-11-11T02:33:13,660 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/4e0dcbfb87684849a5c43f385d6d78ef, entries=150, sequenceid=395, filesize=12.0 K 2024-11-11T02:33:13,661 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for c9be960c11aae9a500eccb669509dc81 in 1245ms, sequenceid=395, compaction requested=false 2024-11-11T02:33:13,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:13,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:13,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-11T02:33:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-11T02:33:13,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-11T02:33:13,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9240 sec 2024-11-11T02:33:13,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.9280 sec 2024-11-11T02:33:13,930 INFO [Thread-1829 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1829. 2024-11-11T02:33:13,931 DEBUG [Thread-1829 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:13,931 DEBUG [Thread-1829 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,931 INFO [Thread-1827 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1827. 2024-11-11T02:33:13,931 DEBUG [Thread-1829 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,931 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:13,931 DEBUG [Thread-1827 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:13,931 DEBUG [Thread-1827 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,931 DEBUG [Thread-1827 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,931 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:13,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-11T02:33:13,934 INFO [Thread-1814 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1814. 2024-11-11T02:33:13,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:33:13,934 DEBUG [Thread-1814 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:13,934 DEBUG [Thread-1814 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:13,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:33:13,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:13,934 DEBUG [Thread-1814 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,934 INFO [Thread-1833 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1833. 2024-11-11T02:33:13,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:13,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:33:13,934 DEBUG [Thread-1833 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:13,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:13,934 DEBUG [Thread-1833 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:13,934 DEBUG [Thread-1833 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,935 INFO [Thread-1820 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1820. 2024-11-11T02:33:13,935 DEBUG [Thread-1820 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:13,935 DEBUG [Thread-1820 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,936 DEBUG [Thread-1820 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:13,936 INFO [Thread-1816 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1816. 2024-11-11T02:33:13,937 DEBUG [Thread-1816 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:13,937 DEBUG [Thread-1816 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,937 DEBUG [Thread-1816 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,937 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:13,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/41b29168573047dc828bc79654f48f5e is 50, key is test_row_0/A:col10/1731292392764/Put/seqid=0 2024-11-11T02:33:13,941 INFO [Thread-1831 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1831. 2024-11-11T02:33:13,941 DEBUG [Thread-1831 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:13,941 DEBUG [Thread-1831 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,941 INFO [Thread-1825 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1825. 2024-11-11T02:33:13,941 DEBUG [Thread-1825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:13,941 DEBUG [Thread-1831 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,941 DEBUG [Thread-1825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,941 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:13,941 DEBUG [Thread-1825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:13,941 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:13,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742291_1467 (size=14741) 2024-11-11T02:33:14,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/41b29168573047dc828bc79654f48f5e 2024-11-11T02:33:14,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/079fe30278c74bf28d4c96f7c554ac6b is 50, key is test_row_0/B:col10/1731292392764/Put/seqid=0 2024-11-11T02:33:14,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742292_1468 (size=12301) 2024-11-11T02:33:14,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/079fe30278c74bf28d4c96f7c554ac6b 2024-11-11T02:33:14,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/794eb4942ff440c1acbc38c498b3db63 is 50, key is test_row_0/C:col10/1731292392764/Put/seqid=0 2024-11-11T02:33:14,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742293_1469 (size=12301) 2024-11-11T02:33:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T02:33:14,906 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:15,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/794eb4942ff440c1acbc38c498b3db63 2024-11-11T02:33:15,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/41b29168573047dc828bc79654f48f5e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/41b29168573047dc828bc79654f48f5e 2024-11-11T02:33:15,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/41b29168573047dc828bc79654f48f5e, entries=200, sequenceid=435, filesize=14.4 K 2024-11-11T02:33:15,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/079fe30278c74bf28d4c96f7c554ac6b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/079fe30278c74bf28d4c96f7c554ac6b 2024-11-11T02:33:15,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/079fe30278c74bf28d4c96f7c554ac6b, entries=150, sequenceid=435, filesize=12.0 K 2024-11-11T02:33:15,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/794eb4942ff440c1acbc38c498b3db63 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/794eb4942ff440c1acbc38c498b3db63 2024-11-11T02:33:15,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/794eb4942ff440c1acbc38c498b3db63, entries=150, sequenceid=435, filesize=12.0 K 2024-11-11T02:33:15,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=13.42 KB/13740 for c9be960c11aae9a500eccb669509dc81 in 1276ms, sequenceid=435, compaction requested=true 2024-11-11T02:33:15,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:15,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:33:15,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:15,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:33:15,210 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:15,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:15,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9be960c11aae9a500eccb669509dc81:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:33:15,210 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:15,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:15,211 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:15,211 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:15,211 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/B is initiating minor compaction (all files) 2024-11-11T02:33:15,211 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/A is initiating minor compaction (all files) 2024-11-11T02:33:15,211 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/B in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:15,211 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/A in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:15,212 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/cb3c41622dcb4b3187fc497ac58b2cae, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/d2a11984b5e34fc39eaa60970bb1bb63, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/41b29168573047dc828bc79654f48f5e] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=39.1 K 2024-11-11T02:33:15,212 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/61af7b6ba736464e8d587ac8a56502bd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/e93b9b70541849328e1abad209ada791, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/079fe30278c74bf28d4c96f7c554ac6b] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=36.7 K 2024-11-11T02:33:15,212 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 61af7b6ba736464e8d587ac8a56502bd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731292390269 2024-11-11T02:33:15,212 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb3c41622dcb4b3187fc497ac58b2cae, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731292390269 2024-11-11T02:33:15,212 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting e93b9b70541849328e1abad209ada791, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731292391471 2024-11-11T02:33:15,212 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2a11984b5e34fc39eaa60970bb1bb63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731292391471 2024-11-11T02:33:15,213 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 079fe30278c74bf28d4c96f7c554ac6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731292392718 2024-11-11T02:33:15,213 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41b29168573047dc828bc79654f48f5e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731292392718 2024-11-11T02:33:15,221 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#B#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:15,221 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#A#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:15,222 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/eb248ae4a4df4581b9cf54bc0eb94e4e is 50, key is test_row_0/B:col10/1731292392764/Put/seqid=0 2024-11-11T02:33:15,222 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/cfed73fb8fa540d9bf27a39904b1320b is 50, key is test_row_0/A:col10/1731292392764/Put/seqid=0 2024-11-11T02:33:15,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742295_1471 (size=13085) 2024-11-11T02:33:15,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742294_1470 (size=13085) 2024-11-11T02:33:15,636 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/eb248ae4a4df4581b9cf54bc0eb94e4e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/eb248ae4a4df4581b9cf54bc0eb94e4e 2024-11-11T02:33:15,636 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/cfed73fb8fa540d9bf27a39904b1320b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/cfed73fb8fa540d9bf27a39904b1320b 2024-11-11T02:33:15,640 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/B of c9be960c11aae9a500eccb669509dc81 into eb248ae4a4df4581b9cf54bc0eb94e4e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:15,640 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/A of c9be960c11aae9a500eccb669509dc81 into cfed73fb8fa540d9bf27a39904b1320b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:15,640 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:15,640 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:15,640 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/B, priority=13, startTime=1731292395210; duration=0sec 2024-11-11T02:33:15,640 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/A, priority=13, startTime=1731292395210; duration=0sec 2024-11-11T02:33:15,640 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:15,640 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:B 2024-11-11T02:33:15,640 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:15,640 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:A 2024-11-11T02:33:15,640 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:15,641 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:15,641 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): c9be960c11aae9a500eccb669509dc81/C is initiating minor compaction (all files) 2024-11-11T02:33:15,641 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9be960c11aae9a500eccb669509dc81/C in TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:15,641 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/2a2670a013324fa2b0d7d797430c1dd9, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/4e0dcbfb87684849a5c43f385d6d78ef, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/794eb4942ff440c1acbc38c498b3db63] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp, totalSize=36.7 K 2024-11-11T02:33:15,641 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a2670a013324fa2b0d7d797430c1dd9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731292390269 2024-11-11T02:33:15,642 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e0dcbfb87684849a5c43f385d6d78ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731292391471 2024-11-11T02:33:15,642 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 794eb4942ff440c1acbc38c498b3db63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731292392718 2024-11-11T02:33:15,648 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9be960c11aae9a500eccb669509dc81#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:15,649 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/aa975ec53d444c3e8b243c32e7bdb987 is 50, key is test_row_0/C:col10/1731292392764/Put/seqid=0 2024-11-11T02:33:15,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742296_1472 (size=13085) 2024-11-11T02:33:16,014 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T02:33:16,064 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/aa975ec53d444c3e8b243c32e7bdb987 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/aa975ec53d444c3e8b243c32e7bdb987 2024-11-11T02:33:16,069 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9be960c11aae9a500eccb669509dc81/C of c9be960c11aae9a500eccb669509dc81 into aa975ec53d444c3e8b243c32e7bdb987(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:16,069 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9be960c11aae9a500eccb669509dc81: 2024-11-11T02:33:16,069 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., storeName=c9be960c11aae9a500eccb669509dc81/C, priority=13, startTime=1731292395210; duration=0sec 2024-11-11T02:33:16,069 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:16,069 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9be960c11aae9a500eccb669509dc81:C 2024-11-11T02:33:21,301 INFO [Thread-1822 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1822. 2024-11-11T02:33:21,301 DEBUG [Thread-1822 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:21,301 DEBUG [Thread-1822 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:21,301 DEBUG [Thread-1822 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:21,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:21,360 INFO [Thread-1818 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1818. 2024-11-11T02:33:21,360 DEBUG [Thread-1818 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:21,360 DEBUG [Thread-1818 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:21,360 DEBUG [Thread-1818 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:21,360 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-11T02:33:21,361 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1188 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3561 rows 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1175 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3523 rows 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1182 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3546 rows 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1187 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3557 rows 2024-11-11T02:33:21,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1175 2024-11-11T02:33:21,362 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3525 rows 2024-11-11T02:33:21,362 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:33:21,362 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:33:21,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:21,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:21,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:21,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T02:33:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:21,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T02:33:21,369 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292401368"}]},"ts":"1731292401368"} 2024-11-11T02:33:21,371 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T02:33:21,371 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T02:33:21,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T02:33:21,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9be960c11aae9a500eccb669509dc81, UNASSIGN}] 2024-11-11T02:33:21,375 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9be960c11aae9a500eccb669509dc81, UNASSIGN 2024-11-11T02:33:21,376 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=c9be960c11aae9a500eccb669509dc81, regionState=CLOSING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:33:21,377 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42785 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=370bc2ade342,38037,1731292247443, table=TestAcidGuarantees, region=c9be960c11aae9a500eccb669509dc81. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T02:33:21,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9be960c11aae9a500eccb669509dc81, UNASSIGN because future has completed 2024-11-11T02:33:21,379 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T02:33:21,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:33:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T02:33:21,532 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:21,532 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T02:33:21,532 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing c9be960c11aae9a500eccb669509dc81, disabling compactions & flushes 2024-11-11T02:33:21,533 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:21,533 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:21,533 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. after waiting 0 ms 2024-11-11T02:33:21,533 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:21,533 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing c9be960c11aae9a500eccb669509dc81 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T02:33:21,534 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=A 2024-11-11T02:33:21,534 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:21,534 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=B 2024-11-11T02:33:21,534 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:21,534 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9be960c11aae9a500eccb669509dc81, store=C 2024-11-11T02:33:21,534 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:21,542 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/51a939e37e6249a1a99b38ea4bd1b71b is 50, key is test_row_0/A:col10/1731292401297/Put/seqid=0 2024-11-11T02:33:21,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742297_1473 (size=12301) 2024-11-11T02:33:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T02:33:21,949 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/51a939e37e6249a1a99b38ea4bd1b71b 2024-11-11T02:33:21,965 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/3fcb408d5c6047f18d72fe9699404a3a is 50, key is test_row_0/B:col10/1731292401297/Put/seqid=0 2024-11-11T02:33:21,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742298_1474 (size=12301) 2024-11-11T02:33:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T02:33:22,370 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/3fcb408d5c6047f18d72fe9699404a3a 2024-11-11T02:33:22,384 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/5819214eaa2441fdb87b58ab893b5803 is 50, key is test_row_0/C:col10/1731292401297/Put/seqid=0 2024-11-11T02:33:22,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742299_1475 (size=12301) 2024-11-11T02:33:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T02:33:22,789 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/5819214eaa2441fdb87b58ab893b5803 2024-11-11T02:33:22,800 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/A/51a939e37e6249a1a99b38ea4bd1b71b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/51a939e37e6249a1a99b38ea4bd1b71b 2024-11-11T02:33:22,805 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/51a939e37e6249a1a99b38ea4bd1b71b, entries=150, sequenceid=445, filesize=12.0 K 2024-11-11T02:33:22,805 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/B/3fcb408d5c6047f18d72fe9699404a3a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/3fcb408d5c6047f18d72fe9699404a3a 2024-11-11T02:33:22,810 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/3fcb408d5c6047f18d72fe9699404a3a, entries=150, sequenceid=445, filesize=12.0 K 2024-11-11T02:33:22,811 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/.tmp/C/5819214eaa2441fdb87b58ab893b5803 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/5819214eaa2441fdb87b58ab893b5803 2024-11-11T02:33:22,816 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/5819214eaa2441fdb87b58ab893b5803, entries=150, sequenceid=445, filesize=12.0 K 2024-11-11T02:33:22,816 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c9be960c11aae9a500eccb669509dc81 in 1283ms, sequenceid=445, compaction requested=false 2024-11-11T02:33:22,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/154b6caa21f347b29e54ac54d408d875, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/e07af46d29e44407a7f55717eaf55954, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/7b4f8156a20d4d57a403e4bf6a34cd9d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/4d7aad04976044618795025c20520ff7, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/6f991537ddd24b51ba4a0e0b6b32d3e0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/cefbb9c85fa54146a450d9b94772e50f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/76062ccd91eb4d38a6f812a4c0ba1813, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/10293f25adfe418e9b2f0fcb8a8862f3, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2cb2c94a7af64a36b46a7dad63ff6133, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2211c1bdcd6e4fbeb2276fb8e55bf193, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/706c7d1688944f1c9059a99f5ee016e4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/9812e3cb908a446a9860fdf84a161f14, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/bf0f35cb34cf419b9c98ed33f92fab55, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/36340238faf241fca697fca6c9cf715e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/7ba2f45ad5fa406d87198da78b2bf154, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/c098f14562ca459a8d25af145f229940, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/a71433d64822466ea75efba25d0545ee, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/f789008954cc4bc4941055a2d73b8b2d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2246cb5ad72d4d6da727f2e4055c402b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/6e83ef2906d34e9ca2ae78d98add9968, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/cb3c41622dcb4b3187fc497ac58b2cae, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/d2a11984b5e34fc39eaa60970bb1bb63, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/41b29168573047dc828bc79654f48f5e] to archive 2024-11-11T02:33:22,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:33:22,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/154b6caa21f347b29e54ac54d408d875 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/154b6caa21f347b29e54ac54d408d875 2024-11-11T02:33:22,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/e07af46d29e44407a7f55717eaf55954 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/e07af46d29e44407a7f55717eaf55954 2024-11-11T02:33:22,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/7b4f8156a20d4d57a403e4bf6a34cd9d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/7b4f8156a20d4d57a403e4bf6a34cd9d 2024-11-11T02:33:22,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/4d7aad04976044618795025c20520ff7 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/4d7aad04976044618795025c20520ff7 2024-11-11T02:33:22,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/6f991537ddd24b51ba4a0e0b6b32d3e0 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/6f991537ddd24b51ba4a0e0b6b32d3e0 2024-11-11T02:33:22,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/cefbb9c85fa54146a450d9b94772e50f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/cefbb9c85fa54146a450d9b94772e50f 2024-11-11T02:33:22,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/76062ccd91eb4d38a6f812a4c0ba1813 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/76062ccd91eb4d38a6f812a4c0ba1813 2024-11-11T02:33:22,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/10293f25adfe418e9b2f0fcb8a8862f3 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/10293f25adfe418e9b2f0fcb8a8862f3 2024-11-11T02:33:22,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2cb2c94a7af64a36b46a7dad63ff6133 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2cb2c94a7af64a36b46a7dad63ff6133 2024-11-11T02:33:22,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2211c1bdcd6e4fbeb2276fb8e55bf193 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2211c1bdcd6e4fbeb2276fb8e55bf193 2024-11-11T02:33:22,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/706c7d1688944f1c9059a99f5ee016e4 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/706c7d1688944f1c9059a99f5ee016e4 2024-11-11T02:33:22,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/9812e3cb908a446a9860fdf84a161f14 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/9812e3cb908a446a9860fdf84a161f14 2024-11-11T02:33:22,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/bf0f35cb34cf419b9c98ed33f92fab55 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/bf0f35cb34cf419b9c98ed33f92fab55 2024-11-11T02:33:22,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/36340238faf241fca697fca6c9cf715e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/36340238faf241fca697fca6c9cf715e 2024-11-11T02:33:22,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/7ba2f45ad5fa406d87198da78b2bf154 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/7ba2f45ad5fa406d87198da78b2bf154 2024-11-11T02:33:22,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/c098f14562ca459a8d25af145f229940 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/c098f14562ca459a8d25af145f229940 2024-11-11T02:33:22,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/a71433d64822466ea75efba25d0545ee to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/a71433d64822466ea75efba25d0545ee 2024-11-11T02:33:22,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/f789008954cc4bc4941055a2d73b8b2d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/f789008954cc4bc4941055a2d73b8b2d 2024-11-11T02:33:22,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2246cb5ad72d4d6da727f2e4055c402b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/2246cb5ad72d4d6da727f2e4055c402b 2024-11-11T02:33:22,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/6e83ef2906d34e9ca2ae78d98add9968 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/6e83ef2906d34e9ca2ae78d98add9968 2024-11-11T02:33:22,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/cb3c41622dcb4b3187fc497ac58b2cae to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/cb3c41622dcb4b3187fc497ac58b2cae 2024-11-11T02:33:22,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/d2a11984b5e34fc39eaa60970bb1bb63 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/d2a11984b5e34fc39eaa60970bb1bb63 2024-11-11T02:33:22,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/41b29168573047dc828bc79654f48f5e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/41b29168573047dc828bc79654f48f5e 2024-11-11T02:33:22,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/a680df97589c4d889a50b5f4fc2f827d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/3835e6ccd84f4f63885b95cf165f394c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/44c1905dde4641a38db38cc9eba29cba, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/2a041be126844b978303fd123630b87d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/03d6a811a9d945a4bbcfd817b3c63a0e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/d241f02a33874a469927c98138b01a7a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/b7c9ecb549a745f0814b7ddefc6c52e3, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/e62766b5e60f4df0ad9cae04e946632b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/8a4486ff1c2249ee91bdd31ffdfc591f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/1a265b1063354aa4bed9e7f10689f83c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/1dd5df47eb24418a8b116563d8536f70, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/901eca06a1c54878b62c97d73a2349e6, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/8671361ca5a24dbd98849746366d6729, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/cade0795171e48babc15f99cf47b4a2b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/9622d0d1b4cd46c49697f7163c130ee6, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/fa47440cd4ff4e85a89f553277648e75, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/82294d8899944584aa42289cb684571f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/d4cf4374035c4a9eb5c4356a1713007f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/2009f570ea8a477b8d5b50d556e1fb65, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/61af7b6ba736464e8d587ac8a56502bd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/7aeab7f73aee485c91b88d375f269eef, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/e93b9b70541849328e1abad209ada791, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/079fe30278c74bf28d4c96f7c554ac6b] to archive 2024-11-11T02:33:22,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:33:22,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/a680df97589c4d889a50b5f4fc2f827d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/a680df97589c4d889a50b5f4fc2f827d 2024-11-11T02:33:22,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/3835e6ccd84f4f63885b95cf165f394c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/3835e6ccd84f4f63885b95cf165f394c 2024-11-11T02:33:22,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/44c1905dde4641a38db38cc9eba29cba to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/44c1905dde4641a38db38cc9eba29cba 2024-11-11T02:33:22,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/2a041be126844b978303fd123630b87d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/2a041be126844b978303fd123630b87d 2024-11-11T02:33:22,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/03d6a811a9d945a4bbcfd817b3c63a0e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/03d6a811a9d945a4bbcfd817b3c63a0e 2024-11-11T02:33:22,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/d241f02a33874a469927c98138b01a7a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/d241f02a33874a469927c98138b01a7a 2024-11-11T02:33:22,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/b7c9ecb549a745f0814b7ddefc6c52e3 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/b7c9ecb549a745f0814b7ddefc6c52e3 2024-11-11T02:33:22,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/e62766b5e60f4df0ad9cae04e946632b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/e62766b5e60f4df0ad9cae04e946632b 2024-11-11T02:33:22,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/8a4486ff1c2249ee91bdd31ffdfc591f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/8a4486ff1c2249ee91bdd31ffdfc591f 2024-11-11T02:33:22,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/1a265b1063354aa4bed9e7f10689f83c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/1a265b1063354aa4bed9e7f10689f83c 2024-11-11T02:33:22,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/1dd5df47eb24418a8b116563d8536f70 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/1dd5df47eb24418a8b116563d8536f70 2024-11-11T02:33:22,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/901eca06a1c54878b62c97d73a2349e6 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/901eca06a1c54878b62c97d73a2349e6 2024-11-11T02:33:22,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/8671361ca5a24dbd98849746366d6729 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/8671361ca5a24dbd98849746366d6729 2024-11-11T02:33:22,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/cade0795171e48babc15f99cf47b4a2b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/cade0795171e48babc15f99cf47b4a2b 2024-11-11T02:33:22,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/9622d0d1b4cd46c49697f7163c130ee6 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/9622d0d1b4cd46c49697f7163c130ee6 2024-11-11T02:33:22,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/fa47440cd4ff4e85a89f553277648e75 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/fa47440cd4ff4e85a89f553277648e75 2024-11-11T02:33:22,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/82294d8899944584aa42289cb684571f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/82294d8899944584aa42289cb684571f 2024-11-11T02:33:22,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/d4cf4374035c4a9eb5c4356a1713007f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/d4cf4374035c4a9eb5c4356a1713007f 2024-11-11T02:33:22,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/2009f570ea8a477b8d5b50d556e1fb65 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/2009f570ea8a477b8d5b50d556e1fb65 2024-11-11T02:33:22,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/61af7b6ba736464e8d587ac8a56502bd to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/61af7b6ba736464e8d587ac8a56502bd 2024-11-11T02:33:22,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/7aeab7f73aee485c91b88d375f269eef to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/7aeab7f73aee485c91b88d375f269eef 2024-11-11T02:33:22,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/e93b9b70541849328e1abad209ada791 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/e93b9b70541849328e1abad209ada791 2024-11-11T02:33:22,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/079fe30278c74bf28d4c96f7c554ac6b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/079fe30278c74bf28d4c96f7c554ac6b 2024-11-11T02:33:22,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/e33c993aee89496f83f4f1329d231ab3, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/317bdd62af45488cbe08ea8e158bce0e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/7bffca6d330a4075bdb66f5c5b7431d5, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/e57614c43e4b458bb2f151b47dba1a60, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/55ff723270e3462b81fdb15890892acc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/22d37c103f544258a12f651860861ba7, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/9d89cedf61ca420a92f5b76b69c48012, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/04c91094f53343879906c6920103dc82, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/ab75cd95d13b49869dcb023168a6b3c2, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0db03edeaa83489796abf84cad058c5b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/6d4b9d2c539e48d9a3613bc5e060cdf6, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/f05d68a718d9465c8729065823ebeb5b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/a56f4f9fb6214e5caa0397291514bab4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/be5cfbbdd1234e2380de599465f3bbdb, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/18eb0b04a02749ea86e8e9d67628fac0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/80d46911e693483d94691364c9ea38d8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0bfab1709cc6466b83a51544af16e2d3, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/330a081c65084d9d939216f674d22814, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0ac791b103f6411685e0fd9c1802a845, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/2a2670a013324fa2b0d7d797430c1dd9, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/d2963b5aa02c4e1cae2b86a272a4cb31, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/4e0dcbfb87684849a5c43f385d6d78ef, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/794eb4942ff440c1acbc38c498b3db63] to archive 2024-11-11T02:33:22,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:33:22,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/e33c993aee89496f83f4f1329d231ab3 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/e33c993aee89496f83f4f1329d231ab3 2024-11-11T02:33:22,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/317bdd62af45488cbe08ea8e158bce0e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/317bdd62af45488cbe08ea8e158bce0e 2024-11-11T02:33:22,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/7bffca6d330a4075bdb66f5c5b7431d5 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/7bffca6d330a4075bdb66f5c5b7431d5 2024-11-11T02:33:22,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/e57614c43e4b458bb2f151b47dba1a60 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/e57614c43e4b458bb2f151b47dba1a60 2024-11-11T02:33:22,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/55ff723270e3462b81fdb15890892acc to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/55ff723270e3462b81fdb15890892acc 2024-11-11T02:33:22,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/22d37c103f544258a12f651860861ba7 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/22d37c103f544258a12f651860861ba7 2024-11-11T02:33:22,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/9d89cedf61ca420a92f5b76b69c48012 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/9d89cedf61ca420a92f5b76b69c48012 2024-11-11T02:33:22,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/04c91094f53343879906c6920103dc82 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/04c91094f53343879906c6920103dc82 2024-11-11T02:33:22,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/ab75cd95d13b49869dcb023168a6b3c2 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/ab75cd95d13b49869dcb023168a6b3c2 2024-11-11T02:33:22,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0db03edeaa83489796abf84cad058c5b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0db03edeaa83489796abf84cad058c5b 2024-11-11T02:33:22,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/6d4b9d2c539e48d9a3613bc5e060cdf6 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/6d4b9d2c539e48d9a3613bc5e060cdf6 2024-11-11T02:33:22,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/f05d68a718d9465c8729065823ebeb5b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/f05d68a718d9465c8729065823ebeb5b 2024-11-11T02:33:22,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/a56f4f9fb6214e5caa0397291514bab4 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/a56f4f9fb6214e5caa0397291514bab4 2024-11-11T02:33:22,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/be5cfbbdd1234e2380de599465f3bbdb to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/be5cfbbdd1234e2380de599465f3bbdb 2024-11-11T02:33:22,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/18eb0b04a02749ea86e8e9d67628fac0 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/18eb0b04a02749ea86e8e9d67628fac0 2024-11-11T02:33:22,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/80d46911e693483d94691364c9ea38d8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/80d46911e693483d94691364c9ea38d8 2024-11-11T02:33:22,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0bfab1709cc6466b83a51544af16e2d3 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0bfab1709cc6466b83a51544af16e2d3 2024-11-11T02:33:22,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/330a081c65084d9d939216f674d22814 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/330a081c65084d9d939216f674d22814 2024-11-11T02:33:22,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0ac791b103f6411685e0fd9c1802a845 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/0ac791b103f6411685e0fd9c1802a845 2024-11-11T02:33:22,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/2a2670a013324fa2b0d7d797430c1dd9 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/2a2670a013324fa2b0d7d797430c1dd9 2024-11-11T02:33:22,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/d2963b5aa02c4e1cae2b86a272a4cb31 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/d2963b5aa02c4e1cae2b86a272a4cb31 2024-11-11T02:33:22,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/4e0dcbfb87684849a5c43f385d6d78ef to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/4e0dcbfb87684849a5c43f385d6d78ef 2024-11-11T02:33:22,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/794eb4942ff440c1acbc38c498b3db63 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/794eb4942ff440c1acbc38c498b3db63 2024-11-11T02:33:22,906 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/recovered.edits/448.seqid, newMaxSeqId=448, maxSeqId=1 2024-11-11T02:33:22,906 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. 2024-11-11T02:33:22,907 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for c9be960c11aae9a500eccb669509dc81: Waiting for close lock at 1731292401532Running coprocessor pre-close hooks at 1731292401532Disabling compacts and flushes for region at 1731292401532Disabling writes for close at 1731292401533 (+1 ms)Obtaining lock to block concurrent updates at 1731292401533Preparing flush snapshotting stores in c9be960c11aae9a500eccb669509dc81 at 1731292401533Finished memstore snapshotting TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731292401534 (+1 ms)Flushing stores of TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81. at 1731292401536 (+2 ms)Flushing c9be960c11aae9a500eccb669509dc81/A: creating writer at 1731292401536Flushing c9be960c11aae9a500eccb669509dc81/A: appending metadata at 1731292401542 (+6 ms)Flushing c9be960c11aae9a500eccb669509dc81/A: closing flushed file at 1731292401542Flushing c9be960c11aae9a500eccb669509dc81/B: creating writer at 1731292401959 (+417 ms)Flushing c9be960c11aae9a500eccb669509dc81/B: appending metadata at 1731292401965 (+6 ms)Flushing c9be960c11aae9a500eccb669509dc81/B: closing flushed file at 1731292401965Flushing c9be960c11aae9a500eccb669509dc81/C: creating writer at 1731292402380 (+415 ms)Flushing c9be960c11aae9a500eccb669509dc81/C: appending metadata at 1731292402383 (+3 ms)Flushing c9be960c11aae9a500eccb669509dc81/C: closing flushed file at 1731292402383Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1699a8b0: reopening flushed file at 1731292402799 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@157bcc00: reopening flushed file at 1731292402805 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6646c30: reopening flushed file at 1731292402810 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c9be960c11aae9a500eccb669509dc81 in 1283ms, sequenceid=445, compaction requested=false at 1731292402816 (+6 ms)Writing region close event to WAL at 1731292402904 (+88 ms)Running coprocessor post-close hooks at 1731292402906 (+2 ms)Closed at 1731292402906 2024-11-11T02:33:22,908 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:22,908 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=c9be960c11aae9a500eccb669509dc81, regionState=CLOSED 2024-11-11T02:33:22,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:33:22,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-11T02:33:22,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure c9be960c11aae9a500eccb669509dc81, server=370bc2ade342,38037,1731292247443 in 1.5310 sec 2024-11-11T02:33:22,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-11T02:33:22,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9be960c11aae9a500eccb669509dc81, UNASSIGN in 1.5380 sec 2024-11-11T02:33:22,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-11T02:33:22,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5420 sec 2024-11-11T02:33:22,916 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292402915"}]},"ts":"1731292402915"} 2024-11-11T02:33:22,917 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T02:33:22,917 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T02:33:22,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5540 sec 2024-11-11T02:33:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T02:33:23,517 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:23,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T02:33:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:23,522 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T02:33:23,523 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:23,527 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:23,530 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/recovered.edits] 2024-11-11T02:33:23,532 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/51a939e37e6249a1a99b38ea4bd1b71b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/51a939e37e6249a1a99b38ea4bd1b71b 2024-11-11T02:33:23,533 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/cfed73fb8fa540d9bf27a39904b1320b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/A/cfed73fb8fa540d9bf27a39904b1320b 2024-11-11T02:33:23,534 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/3fcb408d5c6047f18d72fe9699404a3a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/3fcb408d5c6047f18d72fe9699404a3a 2024-11-11T02:33:23,535 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/eb248ae4a4df4581b9cf54bc0eb94e4e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/B/eb248ae4a4df4581b9cf54bc0eb94e4e 2024-11-11T02:33:23,536 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/5819214eaa2441fdb87b58ab893b5803 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/5819214eaa2441fdb87b58ab893b5803 2024-11-11T02:33:23,537 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/aa975ec53d444c3e8b243c32e7bdb987 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/C/aa975ec53d444c3e8b243c32e7bdb987 2024-11-11T02:33:23,538 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/recovered.edits/448.seqid to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81/recovered.edits/448.seqid 2024-11-11T02:33:23,539 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/c9be960c11aae9a500eccb669509dc81 2024-11-11T02:33:23,539 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T02:33:23,540 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:23,542 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T02:33:23,543 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T02:33:23,544 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:23,544 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T02:33:23,544 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731292403544"}]},"ts":"9223372036854775807"} 2024-11-11T02:33:23,545 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T02:33:23,545 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c9be960c11aae9a500eccb669509dc81, NAME => 'TestAcidGuarantees,,1731292371730.c9be960c11aae9a500eccb669509dc81.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T02:33:23,545 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T02:33:23,545 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731292403545"}]},"ts":"9223372036854775807"} 2024-11-11T02:33:23,546 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T02:33:23,547 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:23,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-11T02:33:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T02:33:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T02:33:23,630 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:23,639 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 248), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=370 (was 437), ProcessCount=11 (was 11), AvailableMemoryMB=5550 (was 5564) 2024-11-11T02:33:23,648 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=370, ProcessCount=11, AvailableMemoryMB=5550 2024-11-11T02:33:23,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T02:33:23,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:33:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:23,652 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T02:33:23,652 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:23,652 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-11-11T02:33:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T02:33:23,653 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T02:33:23,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742300_1476 (size=1061) 2024-11-11T02:33:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T02:33:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T02:33:24,064 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe 2024-11-11T02:33:24,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742301_1477 (size=53) 2024-11-11T02:33:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T02:33:24,476 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:33:24,476 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b190e71023ee9d0d4e9d378db256bf0a, disabling compactions & flushes 2024-11-11T02:33:24,476 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:24,476 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:24,476 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. after waiting 0 ms 2024-11-11T02:33:24,476 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:24,476 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:24,477 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b190e71023ee9d0d4e9d378db256bf0a: Waiting for close lock at 1731292404476Disabling compacts and flushes for region at 1731292404476Disabling writes for close at 1731292404476Writing region close event to WAL at 1731292404476Closed at 1731292404476 2024-11-11T02:33:24,479 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T02:33:24,480 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731292404479"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731292404479"}]},"ts":"1731292404479"} 2024-11-11T02:33:24,484 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T02:33:24,485 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T02:33:24,486 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292404485"}]},"ts":"1731292404485"} 2024-11-11T02:33:24,487 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T02:33:24,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, ASSIGN}] 2024-11-11T02:33:24,488 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, ASSIGN 2024-11-11T02:33:24,489 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, ASSIGN; state=OFFLINE, location=370bc2ade342,38037,1731292247443; forceNewPlan=false, retain=false 2024-11-11T02:33:24,640 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=b190e71023ee9d0d4e9d378db256bf0a, regionState=OPENING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:33:24,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, ASSIGN because future has completed 2024-11-11T02:33:24,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:33:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T02:33:24,807 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:24,807 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:33:24,807 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:24,808 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:33:24,808 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:24,808 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:24,810 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:24,812 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:33:24,812 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b190e71023ee9d0d4e9d378db256bf0a columnFamilyName A 2024-11-11T02:33:24,812 DEBUG [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:24,813 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.HStore(327): Store=b190e71023ee9d0d4e9d378db256bf0a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:33:24,813 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:24,815 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:33:24,815 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b190e71023ee9d0d4e9d378db256bf0a columnFamilyName B 2024-11-11T02:33:24,815 DEBUG [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:24,816 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.HStore(327): Store=b190e71023ee9d0d4e9d378db256bf0a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:33:24,816 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:24,818 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:33:24,818 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b190e71023ee9d0d4e9d378db256bf0a columnFamilyName C 2024-11-11T02:33:24,818 DEBUG [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:24,819 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.HStore(327): Store=b190e71023ee9d0d4e9d378db256bf0a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:33:24,819 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:24,820 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:24,821 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:24,821 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:24,823 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:24,823 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:24,823 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:33:24,825 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:24,827 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:33:24,828 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened b190e71023ee9d0d4e9d378db256bf0a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68276606, jitterRate=0.01740071177482605}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:33:24,828 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:24,829 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for b190e71023ee9d0d4e9d378db256bf0a: Running coprocessor pre-open hook at 1731292404808Writing region info on filesystem at 1731292404808Initializing all the Stores at 1731292404809 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292404809Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292404810 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292404810Cleaning up temporary data from old regions at 1731292404823 (+13 ms)Running coprocessor post-open hooks at 1731292404828 (+5 ms)Region opened successfully at 1731292404828 2024-11-11T02:33:24,829 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., pid=138, masterSystemTime=1731292404800 2024-11-11T02:33:24,831 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:24,831 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:24,832 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=b190e71023ee9d0d4e9d378db256bf0a, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:33:24,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:33:24,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-11T02:33:24,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 in 189 msec 2024-11-11T02:33:24,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-11T02:33:24,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, ASSIGN in 350 msec 2024-11-11T02:33:24,841 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T02:33:24,841 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292404841"}]},"ts":"1731292404841"} 2024-11-11T02:33:24,842 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T02:33:24,843 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T02:33:24,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1930 sec 2024-11-11T02:33:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T02:33:25,798 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:25,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66320673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:25,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:33:25,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:33:25,803 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:33:25,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:33:25,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:33:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e968997, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:33:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:33:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:25,806 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:33:25,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6b1de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:25,807 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:33:25,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:33:25,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:33:25,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:33:25,810 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T02:33:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 370bc2ade342,42785,1731292246873 2024-11-11T02:33:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@70d6591b 2024-11-11T02:33:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T02:33:25,812 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T02:33:25,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T02:33:25,813 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:33:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:25,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742302_1478 (size=1097) 2024-11-11T02:33:26,229 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T02:33:26,229 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T02:33:26,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T02:33:26,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, REOPEN/MOVE}] 2024-11-11T02:33:26,238 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, REOPEN/MOVE 2024-11-11T02:33:26,239 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=b190e71023ee9d0d4e9d378db256bf0a, regionState=CLOSING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:33:26,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, REOPEN/MOVE because future has completed 2024-11-11T02:33:26,240 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T02:33:26,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:33:26,395 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,395 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T02:33:26,395 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing b190e71023ee9d0d4e9d378db256bf0a, disabling compactions & flushes 2024-11-11T02:33:26,395 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:26,395 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:26,395 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. after waiting 0 ms 2024-11-11T02:33:26,395 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:26,404 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T02:33:26,405 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:26,405 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for b190e71023ee9d0d4e9d378db256bf0a: Waiting for close lock at 1731292406395Running coprocessor pre-close hooks at 1731292406395Disabling compacts and flushes for region at 1731292406395Disabling writes for close at 1731292406395Writing region close event to WAL at 1731292406396 (+1 ms)Running coprocessor post-close hooks at 1731292406405 (+9 ms)Closed at 1731292406405 2024-11-11T02:33:26,405 WARN [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: b190e71023ee9d0d4e9d378db256bf0a to self. 2024-11-11T02:33:26,408 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,409 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=b190e71023ee9d0d4e9d378db256bf0a, regionState=CLOSED 2024-11-11T02:33:26,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:33:26,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-11T02:33:26,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 in 172 msec 2024-11-11T02:33:26,415 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, REOPEN/MOVE; state=CLOSED, location=370bc2ade342,38037,1731292247443; forceNewPlan=false, retain=true 2024-11-11T02:33:26,567 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=b190e71023ee9d0d4e9d378db256bf0a, regionState=OPENING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:33:26,571 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, REOPEN/MOVE because future has completed 2024-11-11T02:33:26,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:33:26,732 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:26,732 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:33:26,733 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,733 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:33:26,734 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,734 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,736 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,737 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:33:26,737 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b190e71023ee9d0d4e9d378db256bf0a columnFamilyName A 2024-11-11T02:33:26,738 DEBUG [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:26,738 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.HStore(327): Store=b190e71023ee9d0d4e9d378db256bf0a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:33:26,738 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,739 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:33:26,739 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b190e71023ee9d0d4e9d378db256bf0a columnFamilyName B 2024-11-11T02:33:26,739 DEBUG [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:26,739 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.HStore(327): Store=b190e71023ee9d0d4e9d378db256bf0a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:33:26,740 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,740 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T02:33:26,740 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b190e71023ee9d0d4e9d378db256bf0a columnFamilyName C 2024-11-11T02:33:26,740 DEBUG [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:26,740 INFO [StoreOpener-b190e71023ee9d0d4e9d378db256bf0a-1 {}] regionserver.HStore(327): Store=b190e71023ee9d0d4e9d378db256bf0a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:33:26,741 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:26,741 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,741 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,742 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,743 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,743 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,744 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:33:26,745 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,745 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened b190e71023ee9d0d4e9d378db256bf0a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63933810, jitterRate=-0.047311991453170776}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:33:26,745 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,746 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for b190e71023ee9d0d4e9d378db256bf0a: Running coprocessor pre-open hook at 1731292406734Writing region info on filesystem at 1731292406734Initializing all the Stores at 1731292406735 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292406735Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292406736 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731292406736Cleaning up temporary data from old regions at 1731292406743 (+7 ms)Running coprocessor post-open hooks at 1731292406745 (+2 ms)Region opened successfully at 1731292406746 (+1 ms) 2024-11-11T02:33:26,747 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., pid=143, masterSystemTime=1731292406725 2024-11-11T02:33:26,748 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:26,748 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:26,749 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=b190e71023ee9d0d4e9d378db256bf0a, regionState=OPEN, openSeqNum=5, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:33:26,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:33:26,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-11-11T02:33:26,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 in 180 msec 2024-11-11T02:33:26,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-11T02:33:26,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, REOPEN/MOVE in 516 msec 2024-11-11T02:33:26,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-11T02:33:26,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 522 msec 2024-11-11T02:33:26,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 945 msec 2024-11-11T02:33:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T02:33:26,762 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:26,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:33:26,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:33:26,764 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:33:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:33:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:33:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7675581f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:33:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:33:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:26,766 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:33:26,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24f4464b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:33:26,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:33:26,768 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:33:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:33:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:33:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7813373e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:33:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:33:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:26,769 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:33:26,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f3a7c12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:33:26,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:33:26,771 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:33:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:33:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:33:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@639e1acc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:33:26,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:33:26,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:26,772 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:33:26,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26edf127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:33:26,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:33:26,775 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:33:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:33:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:33:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f476ca4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:33:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:33:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:26,776 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:33:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3093dc40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:33:26,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:33:26,779 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:33:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:33:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:33:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@501df90c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:33:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:33:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:26,780 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:33:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e22f277, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,782 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:33:26,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:33:26,783 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:33:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:33:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:33:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c3ee92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:33:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:33:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:26,784 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:33:26,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@158e1e11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:33:26,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:33:26,786 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:33:26,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:33:26,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:33:26,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6191476c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:33:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:33:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:26,788 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:33:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee90a06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:33:26,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:33:26,790 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:33:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:33:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:33:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c74c6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:33:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:33:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:26,791 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:33:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25546947, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:33:26,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:33:26,794 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:33:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:33:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:33:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a35132f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:33:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:33:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:26,795 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:33:26,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24efc8ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,42785,-1 for getting cluster id 2024-11-11T02:33:26,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:33:26,797 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8a9f923-fca5-401d-96c9-535be72bc124' 2024-11-11T02:33:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:33:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8a9f923-fca5-401d-96c9-535be72bc124" 2024-11-11T02:33:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10804a8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,42785,-1] 2024-11-11T02:33:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:33:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:26,799 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:33:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f0313f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:33:26,800 DEBUG [Thread-2173 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:33:26,800 DEBUG [Thread-2179 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:33:26,800 DEBUG [Thread-2177 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:33:26,800 DEBUG [Thread-2164 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:33:26,800 DEBUG [Thread-2171 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:33:26,800 DEBUG [Thread-2162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:33:26,800 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:33:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:33:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:33:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:33:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:33:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:33:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:33:26,801 DEBUG [Thread-2160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:33:26,801 DEBUG [Thread-2166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:33:26,801 DEBUG [Thread-2175 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:33:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:33:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:33:26,802 DEBUG [Thread-2168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:33:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:33:26,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:33:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:33:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:33:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:33:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:33:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:33:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:33:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:33:26,803 DEBUG [Thread-2171 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:33:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,38037,1731292247443, seqNum=-1] 2024-11-11T02:33:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-11T02:33:26,804 DEBUG [Thread-2171 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:33:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:33:26,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:33:26,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:33:26,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:33:26,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:33:26,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:33:26,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:33:26,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:33:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T02:33:26,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:33:26,805 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:33:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:33:26,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:33:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:33:26,806 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:33:26,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:33:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:33:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:33:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:33:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:33:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:33:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:33:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:33:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5] 2024-11-11T02:33:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:26,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:33:26,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:26,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:26,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:26,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:26,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:26,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:26,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731292406824, max timestamp=1731292406833Num uniques 859; 2024-11-11T02:33:26,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292406825, max timestamp=1731292406833Num uniques 859; 2024-11-11T02:33:26,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:33:26,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T02:33:26,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292406825, max timestamp=1731292406833Num uniques 859; 2024-11-11T02:33:26,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:33:26,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ddb8b806f3d149168bdc9d8bf06c67ed_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292406813/Put/seqid=0 2024-11-11T02:33:26,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742303_1479 (size=17034) 2024-11-11T02:33:26,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40056 deadline: 1731292466852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:26,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292466858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292466860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:26,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:26,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292466862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40050 deadline: 1731292466862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T02:33:26,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T02:33:26,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:26,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:26,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:26,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:26,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:26,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40056 deadline: 1731292466965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:26,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:26,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292466975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292466975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:26,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292466976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40050 deadline: 1731292466976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:27,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T02:33:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:27,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T02:33:27,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40056 deadline: 1731292467175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:27,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292467185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:27,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292467186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292467186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40050 deadline: 1731292467186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:27,255 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:27,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ddb8b806f3d149168bdc9d8bf06c67ed_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ddb8b806f3d149168bdc9d8bf06c67ed_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:27,259 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/722faaa40d2244b5b97f00e5683c453d, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:27,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/722faaa40d2244b5b97f00e5683c453d is 175, key is test_row_0/A:col10/1731292406813/Put/seqid=0 2024-11-11T02:33:27,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742304_1480 (size=48139) 2024-11-11T02:33:27,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T02:33:27,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:27,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:27,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:27,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T02:33:27,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:27,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:27,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:27,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T02:33:27,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40056 deadline: 1731292467485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:27,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:27,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292467495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:27,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292467495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:27,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292467495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:27,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40050 deadline: 1731292467495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:27,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T02:33:27,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:27,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:27,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:27,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,662 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/722faaa40d2244b5b97f00e5683c453d 2024-11-11T02:33:27,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/e818936a008b4ec0894f6d6e468fa920 is 50, key is test_row_0/B:col10/1731292406813/Put/seqid=0 2024-11-11T02:33:27,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742305_1481 (size=12001) 2024-11-11T02:33:27,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T02:33:27,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:27,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:27,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:27,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T02:33:27,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:27,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:27,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:27,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T02:33:27,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40056 deadline: 1731292467995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:27,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:27,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:28,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292468005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:28,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292468005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:28,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292468006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:28,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40050 deadline: 1731292468006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:28,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T02:33:28,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:28,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:28,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:28,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:28,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:28,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/e818936a008b4ec0894f6d6e468fa920 2024-11-11T02:33:28,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/74a1729e095f4453a4a05074a5cd28b0 is 50, key is test_row_0/C:col10/1731292406813/Put/seqid=0 2024-11-11T02:33:28,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742306_1482 (size=12001) 2024-11-11T02:33:28,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T02:33:28,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:28,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:28,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:28,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:28,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:28,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T02:33:28,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:28,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:28,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:28,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:28,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:28,423 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T02:33:28,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T02:33:28,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:28,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:28,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:28,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:28,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:28,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/74a1729e095f4453a4a05074a5cd28b0 2024-11-11T02:33:28,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/722faaa40d2244b5b97f00e5683c453d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/722faaa40d2244b5b97f00e5683c453d 2024-11-11T02:33:28,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/722faaa40d2244b5b97f00e5683c453d, entries=250, sequenceid=15, filesize=47.0 K 2024-11-11T02:33:28,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/e818936a008b4ec0894f6d6e468fa920 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/e818936a008b4ec0894f6d6e468fa920 2024-11-11T02:33:28,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/e818936a008b4ec0894f6d6e468fa920, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T02:33:28,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/74a1729e095f4453a4a05074a5cd28b0 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/74a1729e095f4453a4a05074a5cd28b0 2024-11-11T02:33:28,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/74a1729e095f4453a4a05074a5cd28b0, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T02:33:28,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for b190e71023ee9d0d4e9d378db256bf0a in 1706ms, sequenceid=15, compaction requested=false 2024-11-11T02:33:28,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:28,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T02:33:28,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:28,638 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-11-11T02:33:28,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:28,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:28,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:28,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:28,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:28,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:28,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111104d096fe65f946ce85cbb3ef35a03b01_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292406850/Put/seqid=0 2024-11-11T02:33:28,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742307_1483 (size=12154) 2024-11-11T02:33:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T02:33:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:29,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:29,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40050 deadline: 1731292469024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:29,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:29,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292469025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:29,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:29,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292469026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:29,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292469026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:29,029 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:29,029 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:29,029 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:29,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:29,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40056 deadline: 1731292469028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:29,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:29,052 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111104d096fe65f946ce85cbb3ef35a03b01_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111104d096fe65f946ce85cbb3ef35a03b01_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:29,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/680de6e4d828413daddfec267cb0bd6b, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:29,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/680de6e4d828413daddfec267cb0bd6b is 175, key is test_row_0/A:col10/1731292406850/Put/seqid=0 2024-11-11T02:33:29,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742308_1484 (size=30955) 2024-11-11T02:33:29,057 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/680de6e4d828413daddfec267cb0bd6b 2024-11-11T02:33:29,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/677299855b1b4661a8e1b0ebfb1e575f is 50, key is test_row_0/B:col10/1731292406850/Put/seqid=0 2024-11-11T02:33:29,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742309_1485 (size=12001) 2024-11-11T02:33:29,066 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/677299855b1b4661a8e1b0ebfb1e575f 2024-11-11T02:33:29,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/b18724c4175945b188ff846e25ad3755 is 50, key is test_row_0/C:col10/1731292406850/Put/seqid=0 2024-11-11T02:33:29,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742310_1486 (size=12001) 2024-11-11T02:33:29,075 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/b18724c4175945b188ff846e25ad3755 2024-11-11T02:33:29,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/680de6e4d828413daddfec267cb0bd6b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/680de6e4d828413daddfec267cb0bd6b 2024-11-11T02:33:29,082 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/680de6e4d828413daddfec267cb0bd6b, entries=150, sequenceid=51, filesize=30.2 K 2024-11-11T02:33:29,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/677299855b1b4661a8e1b0ebfb1e575f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/677299855b1b4661a8e1b0ebfb1e575f 2024-11-11T02:33:29,085 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/677299855b1b4661a8e1b0ebfb1e575f, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T02:33:29,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/b18724c4175945b188ff846e25ad3755 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b18724c4175945b188ff846e25ad3755 2024-11-11T02:33:29,089 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b18724c4175945b188ff846e25ad3755, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T02:33:29,090 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=46.96 KB/48090 for b190e71023ee9d0d4e9d378db256bf0a in 452ms, sequenceid=51, compaction requested=false 2024-11-11T02:33:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-11T02:33:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-11T02:33:29,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-11T02:33:29,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2860 sec 2024-11-11T02:33:29,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.2930 sec 2024-11-11T02:33:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:29,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T02:33:29,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:29,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:29,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:29,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:29,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:29,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:29,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bc2176c57f35487caea541d00acac16b_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292409022/Put/seqid=0 2024-11-11T02:33:29,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742311_1487 (size=14594) 2024-11-11T02:33:29,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40056 deadline: 1731292469180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:29,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:29,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40050 deadline: 1731292469184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:29,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40056 deadline: 1731292469295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:29,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:29,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40050 deadline: 1731292469295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:29,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:29,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40056 deadline: 1731292469506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40050 deadline: 1731292469506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:29,553 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:29,556 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bc2176c57f35487caea541d00acac16b_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bc2176c57f35487caea541d00acac16b_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:29,557 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/9eea0ade92474967b3f46f3b3f14944a, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:29,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/9eea0ade92474967b3f46f3b3f14944a is 175, key is test_row_0/A:col10/1731292409022/Put/seqid=0 2024-11-11T02:33:29,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742312_1488 (size=39549) 2024-11-11T02:33:29,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40056 deadline: 1731292469815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:29,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40050 deadline: 1731292469815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:29,961 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/9eea0ade92474967b3f46f3b3f14944a 2024-11-11T02:33:29,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/5cbf597b0b614233a674875fe6487519 is 50, key is test_row_0/B:col10/1731292409022/Put/seqid=0 2024-11-11T02:33:29,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742313_1489 (size=12001) 2024-11-11T02:33:30,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40056 deadline: 1731292470325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:30,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40050 deadline: 1731292470325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:30,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:30,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:30,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:30,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:30,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:30,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:30,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/5cbf597b0b614233a674875fe6487519 2024-11-11T02:33:30,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/fef147dde01a47c6912c8d4e03a3b05b is 50, key is test_row_0/C:col10/1731292409022/Put/seqid=0 2024-11-11T02:33:30,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742314_1490 (size=12001) 2024-11-11T02:33:30,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/fef147dde01a47c6912c8d4e03a3b05b 2024-11-11T02:33:30,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/9eea0ade92474967b3f46f3b3f14944a as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/9eea0ade92474967b3f46f3b3f14944a 2024-11-11T02:33:30,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/9eea0ade92474967b3f46f3b3f14944a, entries=200, sequenceid=63, filesize=38.6 K 2024-11-11T02:33:30,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/5cbf597b0b614233a674875fe6487519 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/5cbf597b0b614233a674875fe6487519 2024-11-11T02:33:30,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/5cbf597b0b614233a674875fe6487519, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T02:33:30,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/fef147dde01a47c6912c8d4e03a3b05b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/fef147dde01a47c6912c8d4e03a3b05b 2024-11-11T02:33:30,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/fef147dde01a47c6912c8d4e03a3b05b, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T02:33:30,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b190e71023ee9d0d4e9d378db256bf0a in 1658ms, sequenceid=63, compaction requested=true 2024-11-11T02:33:30,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:30,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:33:30,796 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:30,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:30,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:33:30,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:30,796 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:30,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:33:30,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:30,796 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:30,796 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:30,796 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/A is initiating minor compaction (all files) 2024-11-11T02:33:30,796 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/B is initiating minor compaction (all files) 2024-11-11T02:33:30,797 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/B in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:30,797 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/A in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:30,797 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/722faaa40d2244b5b97f00e5683c453d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/680de6e4d828413daddfec267cb0bd6b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/9eea0ade92474967b3f46f3b3f14944a] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=115.9 K 2024-11-11T02:33:30,797 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/e818936a008b4ec0894f6d6e468fa920, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/677299855b1b4661a8e1b0ebfb1e575f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/5cbf597b0b614233a674875fe6487519] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=35.2 K 2024-11-11T02:33:30,797 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:30,797 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/722faaa40d2244b5b97f00e5683c453d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/680de6e4d828413daddfec267cb0bd6b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/9eea0ade92474967b3f46f3b3f14944a] 2024-11-11T02:33:30,797 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting e818936a008b4ec0894f6d6e468fa920, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731292406811 2024-11-11T02:33:30,797 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 722faaa40d2244b5b97f00e5683c453d, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731292406808 2024-11-11T02:33:30,797 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 677299855b1b4661a8e1b0ebfb1e575f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731292406850 2024-11-11T02:33:30,797 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 680de6e4d828413daddfec267cb0bd6b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731292406850 2024-11-11T02:33:30,797 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cbf597b0b614233a674875fe6487519, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731292409022 2024-11-11T02:33:30,797 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eea0ade92474967b3f46f3b3f14944a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731292409022 2024-11-11T02:33:30,802 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:30,803 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:30,803 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/9cda936b44204d49b90962784043e33e is 50, key is test_row_0/B:col10/1731292409022/Put/seqid=0 2024-11-11T02:33:30,805 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411118aba2600375f41159571c585e9425826_b190e71023ee9d0d4e9d378db256bf0a store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:30,807 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411118aba2600375f41159571c585e9425826_b190e71023ee9d0d4e9d378db256bf0a, store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:30,808 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118aba2600375f41159571c585e9425826_b190e71023ee9d0d4e9d378db256bf0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:30,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742315_1491 (size=12104) 2024-11-11T02:33:30,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742316_1492 (size=4469) 2024-11-11T02:33:30,839 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#A#compaction#412 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:30,839 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/ec69de021fa346c5936e037215d581fd is 175, key is test_row_0/A:col10/1731292409022/Put/seqid=0 2024-11-11T02:33:30,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742317_1493 (size=31058) 2024-11-11T02:33:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T02:33:30,966 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:30,966 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:33:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-11T02:33:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T02:33:30,968 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:33:30,968 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:33:30,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:33:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:31,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T02:33:31,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:31,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:31,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:31,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:31,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:31,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:31,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c3f56c9c86a7431b9cde40957af82766_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292409168/Put/seqid=0 2024-11-11T02:33:31,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742318_1494 (size=14594) 2024-11-11T02:33:31,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292471063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:31,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292471065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T02:33:31,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292471069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:31,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T02:33:31,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:31,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:31,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:31,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292471175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:31,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:31,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292471175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:31,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:31,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:31,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292471185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:31,234 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/9cda936b44204d49b90962784043e33e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/9cda936b44204d49b90962784043e33e 2024-11-11T02:33:31,237 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/B of b190e71023ee9d0d4e9d378db256bf0a into 9cda936b44204d49b90962784043e33e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:31,237 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:31,237 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/B, priority=13, startTime=1731292410796; duration=0sec 2024-11-11T02:33:31,237 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:31,237 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:B 2024-11-11T02:33:31,237 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:31,238 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:31,238 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/C is initiating minor compaction (all files) 2024-11-11T02:33:31,238 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/C in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:31,239 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/74a1729e095f4453a4a05074a5cd28b0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b18724c4175945b188ff846e25ad3755, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/fef147dde01a47c6912c8d4e03a3b05b] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=35.2 K 2024-11-11T02:33:31,239 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 74a1729e095f4453a4a05074a5cd28b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731292406811 2024-11-11T02:33:31,239 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting b18724c4175945b188ff846e25ad3755, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731292406850 2024-11-11T02:33:31,239 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting fef147dde01a47c6912c8d4e03a3b05b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731292409022 2024-11-11T02:33:31,245 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#C#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:31,246 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/2e7a598d3e6f4f06baef6a8ec0b8e675 is 50, key is test_row_0/C:col10/1731292409022/Put/seqid=0 2024-11-11T02:33:31,252 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/ec69de021fa346c5936e037215d581fd as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ec69de021fa346c5936e037215d581fd 2024-11-11T02:33:31,255 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/A of b190e71023ee9d0d4e9d378db256bf0a into ec69de021fa346c5936e037215d581fd(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:31,255 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:31,255 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/A, priority=13, startTime=1731292410796; duration=0sec 2024-11-11T02:33:31,256 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:31,256 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:A 2024-11-11T02:33:31,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742319_1495 (size=12104) 2024-11-11T02:33:31,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T02:33:31,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:31,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:31,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:31,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T02:33:31,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40056 deadline: 1731292471335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:31,337 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:31,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40050 deadline: 1731292471345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:31,350 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:31,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292471385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:31,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292471385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:31,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292471395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:31,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:31,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T02:33:31,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:31,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:31,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:31,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,458 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:31,461 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c3f56c9c86a7431b9cde40957af82766_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c3f56c9c86a7431b9cde40957af82766_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:31,461 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/6659270505094f479e4de854c91a2954, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:31,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/6659270505094f479e4de854c91a2954 is 175, key is test_row_0/A:col10/1731292409168/Put/seqid=0 2024-11-11T02:33:31,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742320_1496 (size=39549) 2024-11-11T02:33:31,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T02:33:31,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:31,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:31,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:31,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T02:33:31,666 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/2e7a598d3e6f4f06baef6a8ec0b8e675 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/2e7a598d3e6f4f06baef6a8ec0b8e675 2024-11-11T02:33:31,669 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/C of b190e71023ee9d0d4e9d378db256bf0a into 2e7a598d3e6f4f06baef6a8ec0b8e675(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:31,669 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:31,669 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/C, priority=13, startTime=1731292410796; duration=0sec 2024-11-11T02:33:31,669 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:31,669 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:C 2024-11-11T02:33:31,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292471695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292471696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:31,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292471706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:31,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T02:33:31,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:31,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:31,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:31,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,865 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/6659270505094f479e4de854c91a2954 2024-11-11T02:33:31,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T02:33:31,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:31,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:31,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:31,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/3b072248103640be98db3b75a247615c is 50, key is test_row_0/B:col10/1731292409168/Put/seqid=0 2024-11-11T02:33:31,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:31,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742321_1497 (size=12001) 2024-11-11T02:33:32,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T02:33:32,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:32,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:32,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:32,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T02:33:32,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T02:33:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:32,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292472205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:32,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292472206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:32,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:32,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:32,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:32,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:32,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:32,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:32,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292472216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:32,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/3b072248103640be98db3b75a247615c 2024-11-11T02:33:32,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/c04157731e9f4f73afa206b2bccd10f8 is 50, key is test_row_0/C:col10/1731292409168/Put/seqid=0 2024-11-11T02:33:32,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742322_1498 (size=12001) 2024-11-11T02:33:32,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T02:33:32,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:32,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:32,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:32,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T02:33:32,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:32,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:32,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:32,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T02:33:32,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:32,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:32,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:32,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:32,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/c04157731e9f4f73afa206b2bccd10f8 2024-11-11T02:33:32,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/6659270505094f479e4de854c91a2954 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/6659270505094f479e4de854c91a2954 2024-11-11T02:33:32,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/6659270505094f479e4de854c91a2954, entries=200, sequenceid=84, filesize=38.6 K 2024-11-11T02:33:32,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/3b072248103640be98db3b75a247615c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/3b072248103640be98db3b75a247615c 2024-11-11T02:33:32,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/3b072248103640be98db3b75a247615c, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T02:33:32,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/c04157731e9f4f73afa206b2bccd10f8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/c04157731e9f4f73afa206b2bccd10f8 2024-11-11T02:33:32,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/c04157731e9f4f73afa206b2bccd10f8, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T02:33:32,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for b190e71023ee9d0d4e9d378db256bf0a in 1662ms, sequenceid=84, compaction requested=false 2024-11-11T02:33:32,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:32,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T02:33:32,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:32,799 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T02:33:32,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:32,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:32,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:32,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:32,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:32,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:32,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112e61c42f3fe149fc9bda16ad81309c3b_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292411053/Put/seqid=0 2024-11-11T02:33:32,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742323_1499 (size=12154) 2024-11-11T02:33:33,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T02:33:33,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:33,215 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112e61c42f3fe149fc9bda16ad81309c3b_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112e61c42f3fe149fc9bda16ad81309c3b_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:33,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/a238186e1d1c408396a9d3bf4d2240c8, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:33,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/a238186e1d1c408396a9d3bf4d2240c8 is 175, key is test_row_0/A:col10/1731292411053/Put/seqid=0 2024-11-11T02:33:33,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:33,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:33,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742324_1500 (size=30955) 2024-11-11T02:33:33,220 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/a238186e1d1c408396a9d3bf4d2240c8 2024-11-11T02:33:33,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/97579b510d3d4381aab06db635380351 is 50, key is test_row_0/B:col10/1731292411053/Put/seqid=0 2024-11-11T02:33:33,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742325_1501 (size=12001) 2024-11-11T02:33:33,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731292413267, max timestamp=1731292413279Num uniques 859; 2024-11-11T02:33:33,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T02:33:33,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731292413267, max timestamp=1731292413279Num uniques 859; 2024-11-11T02:33:33,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T02:33:33,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731292413267, max timestamp=1731292413279Num uniques 859; 2024-11-11T02:33:33,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T02:33:33,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292473306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:33,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292473307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:33,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:33,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292473309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:33,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:33,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:33,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40056 deadline: 1731292473346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:33,350 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:33,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40050 deadline: 1731292473365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:33,369 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:33,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292473416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:33,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292473416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:33,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:33,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292473416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:33,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292473625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:33,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292473625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292473626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:33,630 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/97579b510d3d4381aab06db635380351 2024-11-11T02:33:33,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/f629fa552a3948b4bb4482245807f392 is 50, key is test_row_0/C:col10/1731292411053/Put/seqid=0 2024-11-11T02:33:33,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742326_1502 (size=12001) 2024-11-11T02:33:33,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292473935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292473935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292473935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:34,039 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/f629fa552a3948b4bb4482245807f392 2024-11-11T02:33:34,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/a238186e1d1c408396a9d3bf4d2240c8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/a238186e1d1c408396a9d3bf4d2240c8 2024-11-11T02:33:34,045 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/a238186e1d1c408396a9d3bf4d2240c8, entries=150, sequenceid=98, filesize=30.2 K 2024-11-11T02:33:34,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/97579b510d3d4381aab06db635380351 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/97579b510d3d4381aab06db635380351 2024-11-11T02:33:34,048 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/97579b510d3d4381aab06db635380351, entries=150, sequenceid=98, filesize=11.7 K 2024-11-11T02:33:34,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/f629fa552a3948b4bb4482245807f392 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/f629fa552a3948b4bb4482245807f392 2024-11-11T02:33:34,051 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/f629fa552a3948b4bb4482245807f392, entries=150, sequenceid=98, filesize=11.7 K 2024-11-11T02:33:34,052 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b190e71023ee9d0d4e9d378db256bf0a in 1253ms, sequenceid=98, compaction requested=true 2024-11-11T02:33:34,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:34,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:34,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-11T02:33:34,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-11T02:33:34,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-11T02:33:34,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0840 sec 2024-11-11T02:33:34,056 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 3.0880 sec 2024-11-11T02:33:34,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:34,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-11T02:33:34,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:34,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:34,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:34,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:34,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:34,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:34,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111909f4ce759fa49cda80ecbeb4923ea82_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292413308/Put/seqid=0 2024-11-11T02:33:34,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742327_1503 (size=14744) 2024-11-11T02:33:34,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292474466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292474468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:34,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292474468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:34,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292474575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:34,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292474575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:34,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292474586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:34,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292474786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292474786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292474795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:34,859 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:34,862 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111909f4ce759fa49cda80ecbeb4923ea82_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111909f4ce759fa49cda80ecbeb4923ea82_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:34,863 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/0e10f96ddf0a47e690d16bb638967e08, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:34,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/0e10f96ddf0a47e690d16bb638967e08 is 175, key is test_row_0/A:col10/1731292413308/Put/seqid=0 2024-11-11T02:33:34,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742328_1504 (size=39699) 2024-11-11T02:33:35,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292475095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:35,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292475095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:35,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292475105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T02:33:35,126 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:35,126 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:33:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-11T02:33:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T02:33:35,128 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:33:35,129 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:33:35,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:33:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T02:33:35,267 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/0e10f96ddf0a47e690d16bb638967e08 2024-11-11T02:33:35,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/89a9cfa819e8462699bd23610ac431a2 is 50, key is test_row_0/B:col10/1731292413308/Put/seqid=0 2024-11-11T02:33:35,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742329_1505 (size=12151) 2024-11-11T02:33:35,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T02:33:35,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:35,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:35,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:35,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:35,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:35,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T02:33:35,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:35,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:35,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:35,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:35,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T02:33:35,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T02:33:35,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:35,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:35,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:35,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:35,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:35,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292475605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:35,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292475605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:35,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292475616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:35,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/89a9cfa819e8462699bd23610ac431a2 2024-11-11T02:33:35,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/b08b13bd32ad4884aeba2bd051759fd8 is 50, key is test_row_0/C:col10/1731292413308/Put/seqid=0 2024-11-11T02:33:35,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742330_1506 (size=12151) 2024-11-11T02:33:35,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T02:33:35,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:35,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:35,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:35,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:35,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T02:33:35,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T02:33:35,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:35,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:35,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:35,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:35,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:36,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T02:33:36,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:36,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:36,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:36,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:36,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:36,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/b08b13bd32ad4884aeba2bd051759fd8 2024-11-11T02:33:36,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/0e10f96ddf0a47e690d16bb638967e08 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/0e10f96ddf0a47e690d16bb638967e08 2024-11-11T02:33:36,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/0e10f96ddf0a47e690d16bb638967e08, entries=200, sequenceid=133, filesize=38.8 K 2024-11-11T02:33:36,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/89a9cfa819e8462699bd23610ac431a2 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/89a9cfa819e8462699bd23610ac431a2 2024-11-11T02:33:36,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/89a9cfa819e8462699bd23610ac431a2, entries=150, sequenceid=133, filesize=11.9 K 2024-11-11T02:33:36,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/b08b13bd32ad4884aeba2bd051759fd8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b08b13bd32ad4884aeba2bd051759fd8 2024-11-11T02:33:36,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b08b13bd32ad4884aeba2bd051759fd8, entries=150, sequenceid=133, filesize=11.9 K 2024-11-11T02:33:36,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for b190e71023ee9d0d4e9d378db256bf0a in 1651ms, sequenceid=133, compaction requested=true 2024-11-11T02:33:36,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:36,101 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:33:36,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:33:36,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:36,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:33:36,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:36,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:33:36,101 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:33:36,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:36,102 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141261 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:33:36,102 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:33:36,102 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/A is initiating minor compaction (all files) 2024-11-11T02:33:36,102 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/B is initiating minor compaction (all files) 2024-11-11T02:33:36,102 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/B in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:36,102 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/A in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:36,102 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/9cda936b44204d49b90962784043e33e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/3b072248103640be98db3b75a247615c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/97579b510d3d4381aab06db635380351, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/89a9cfa819e8462699bd23610ac431a2] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=47.1 K 2024-11-11T02:33:36,102 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ec69de021fa346c5936e037215d581fd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/6659270505094f479e4de854c91a2954, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/a238186e1d1c408396a9d3bf4d2240c8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/0e10f96ddf0a47e690d16bb638967e08] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=138.0 K 2024-11-11T02:33:36,102 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:36,102 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ec69de021fa346c5936e037215d581fd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/6659270505094f479e4de854c91a2954, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/a238186e1d1c408396a9d3bf4d2240c8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/0e10f96ddf0a47e690d16bb638967e08] 2024-11-11T02:33:36,103 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cda936b44204d49b90962784043e33e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731292409022 2024-11-11T02:33:36,103 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec69de021fa346c5936e037215d581fd, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731292409022 2024-11-11T02:33:36,103 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b072248103640be98db3b75a247615c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731292409160 2024-11-11T02:33:36,103 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6659270505094f479e4de854c91a2954, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731292409160 2024-11-11T02:33:36,103 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 97579b510d3d4381aab06db635380351, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731292411053 2024-11-11T02:33:36,103 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting a238186e1d1c408396a9d3bf4d2240c8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731292411053 2024-11-11T02:33:36,103 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 89a9cfa819e8462699bd23610ac431a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731292413305 2024-11-11T02:33:36,103 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e10f96ddf0a47e690d16bb638967e08, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731292413291 2024-11-11T02:33:36,108 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:36,110 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:36,110 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411118780a409aed24b0d8e9ffc5f298f6ca2_b190e71023ee9d0d4e9d378db256bf0a store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:36,110 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/70dcda13439b48c09a3af657f6f05b3f is 50, key is test_row_0/B:col10/1731292413308/Put/seqid=0 2024-11-11T02:33:36,112 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411118780a409aed24b0d8e9ffc5f298f6ca2_b190e71023ee9d0d4e9d378db256bf0a, store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:36,112 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118780a409aed24b0d8e9ffc5f298f6ca2_b190e71023ee9d0d4e9d378db256bf0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:36,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742331_1507 (size=12391) 2024-11-11T02:33:36,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742332_1508 (size=4469) 2024-11-11T02:33:36,127 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#A#compaction#424 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:36,128 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/e23bf1c520854d81b12b651f97fe022f is 175, key is test_row_0/A:col10/1731292413308/Put/seqid=0 2024-11-11T02:33:36,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742333_1509 (size=31345) 2024-11-11T02:33:36,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T02:33:36,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:36,196 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T02:33:36,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:36,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:36,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:36,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:36,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:36,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118b54af7a747f4045bc262688c8f0e692_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292414467/Put/seqid=0 2024-11-11T02:33:36,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742334_1510 (size=12304) 2024-11-11T02:33:36,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:36,209 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118b54af7a747f4045bc262688c8f0e692_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118b54af7a747f4045bc262688c8f0e692_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:36,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/09068d3dadbd4dafa676e17dd787a1fc, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:36,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/09068d3dadbd4dafa676e17dd787a1fc is 175, key is test_row_0/A:col10/1731292414467/Put/seqid=0 2024-11-11T02:33:36,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742335_1511 (size=31105) 2024-11-11T02:33:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T02:33:36,522 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/70dcda13439b48c09a3af657f6f05b3f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/70dcda13439b48c09a3af657f6f05b3f 2024-11-11T02:33:36,526 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/B of b190e71023ee9d0d4e9d378db256bf0a into 70dcda13439b48c09a3af657f6f05b3f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:36,526 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:36,526 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/B, priority=12, startTime=1731292416101; duration=0sec 2024-11-11T02:33:36,526 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:36,526 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:B 2024-11-11T02:33:36,526 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T02:33:36,528 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T02:33:36,528 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/C is initiating minor compaction (all files) 2024-11-11T02:33:36,528 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/C in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:36,528 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/2e7a598d3e6f4f06baef6a8ec0b8e675, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/c04157731e9f4f73afa206b2bccd10f8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/f629fa552a3948b4bb4482245807f392, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b08b13bd32ad4884aeba2bd051759fd8] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=47.1 K 2024-11-11T02:33:36,528 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e7a598d3e6f4f06baef6a8ec0b8e675, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731292409022 2024-11-11T02:33:36,528 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting c04157731e9f4f73afa206b2bccd10f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731292409160 2024-11-11T02:33:36,529 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting f629fa552a3948b4bb4482245807f392, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731292411053 2024-11-11T02:33:36,529 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting b08b13bd32ad4884aeba2bd051759fd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731292413305 2024-11-11T02:33:36,535 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/e23bf1c520854d81b12b651f97fe022f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/e23bf1c520854d81b12b651f97fe022f 2024-11-11T02:33:36,537 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#C#compaction#426 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:36,537 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/42db843f642644f19b38344ef56bec43 is 50, key is test_row_0/C:col10/1731292413308/Put/seqid=0 2024-11-11T02:33:36,542 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/A of b190e71023ee9d0d4e9d378db256bf0a into e23bf1c520854d81b12b651f97fe022f(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:36,542 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:36,542 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/A, priority=12, startTime=1731292416101; duration=0sec 2024-11-11T02:33:36,542 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:36,542 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:A 2024-11-11T02:33:36,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742336_1512 (size=12391) 2024-11-11T02:33:36,615 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/09068d3dadbd4dafa676e17dd787a1fc 2024-11-11T02:33:36,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/c9c38a7ab9654347b5bc703e4e38fcaf is 50, key is test_row_0/B:col10/1731292414467/Put/seqid=0 2024-11-11T02:33:36,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742337_1513 (size=12151) 2024-11-11T02:33:36,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:36,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292476656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:36,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:36,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:36,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:36,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292416646, max timestamp=1731292416659Num uniques 859; 2024-11-11T02:33:36,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:36,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292416646, max timestamp=1731292416659Num uniques 859; 2024-11-11T02:33:36,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:36,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731292416646, max timestamp=1731292416659Num uniques 859; 2024-11-11T02:33:36,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T02:33:36,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292476699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:36,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:36,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:36,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:36,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292476702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:36,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292476765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:36,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292476816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:36,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292476816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:36,952 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/42db843f642644f19b38344ef56bec43 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/42db843f642644f19b38344ef56bec43 2024-11-11T02:33:36,955 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/C of b190e71023ee9d0d4e9d378db256bf0a into 42db843f642644f19b38344ef56bec43(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:36,955 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:36,955 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/C, priority=12, startTime=1731292416101; duration=0sec 2024-11-11T02:33:36,955 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:36,955 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:C 2024-11-11T02:33:36,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292476975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:37,025 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/c9c38a7ab9654347b5bc703e4e38fcaf 2024-11-11T02:33:37,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292477026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:37,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292477026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:37,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/abdcd9cbec6e4ee6ac6df6519f453721 is 50, key is test_row_0/C:col10/1731292414467/Put/seqid=0 2024-11-11T02:33:37,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742338_1514 (size=12151) 2024-11-11T02:33:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T02:33:37,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292477286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:37,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292477335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:37,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292477335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:37,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40056 deadline: 1731292477356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:37,359 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:37,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:37,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40050 deadline: 1731292477385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:37,388 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_2' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:37,440 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/abdcd9cbec6e4ee6ac6df6519f453721 2024-11-11T02:33:37,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/09068d3dadbd4dafa676e17dd787a1fc as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/09068d3dadbd4dafa676e17dd787a1fc 2024-11-11T02:33:37,446 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/09068d3dadbd4dafa676e17dd787a1fc, entries=150, sequenceid=144, filesize=30.4 K 2024-11-11T02:33:37,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/c9c38a7ab9654347b5bc703e4e38fcaf as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c9c38a7ab9654347b5bc703e4e38fcaf 2024-11-11T02:33:37,449 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c9c38a7ab9654347b5bc703e4e38fcaf, entries=150, sequenceid=144, filesize=11.9 K 2024-11-11T02:33:37,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/abdcd9cbec6e4ee6ac6df6519f453721 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/abdcd9cbec6e4ee6ac6df6519f453721 2024-11-11T02:33:37,452 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/abdcd9cbec6e4ee6ac6df6519f453721, entries=150, sequenceid=144, filesize=11.9 K 2024-11-11T02:33:37,453 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b190e71023ee9d0d4e9d378db256bf0a in 1257ms, sequenceid=144, compaction requested=false 2024-11-11T02:33:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-11T02:33:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-11T02:33:37,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-11T02:33:37,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3250 sec 2024-11-11T02:33:37,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.3290 sec 2024-11-11T02:33:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:37,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T02:33:37,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:37,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:37,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:37,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:37,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:37,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T02:33:37,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111006a4e739bad46b2b4a74e2d6e77f8ac_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292416700/Put/seqid=0 2024-11-11T02:33:37,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742339_1515 (size=12304) 2024-11-11T02:33:37,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292477839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:37,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292477845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:37,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292477845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:37,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292477945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:38,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:38,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292478156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:38,211 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:38,214 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111006a4e739bad46b2b4a74e2d6e77f8ac_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111006a4e739bad46b2b4a74e2d6e77f8ac_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:38,215 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/be99ec5ebfeb446d945623717d3885fd, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:38,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/be99ec5ebfeb446d945623717d3885fd is 175, key is test_row_0/A:col10/1731292416700/Put/seqid=0 2024-11-11T02:33:38,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742340_1516 (size=31105) 2024-11-11T02:33:38,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292478465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:38,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:38,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:38,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:38,619 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/be99ec5ebfeb446d945623717d3885fd 2024-11-11T02:33:38,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/c122022e0018414d95674a1c74b94b46 is 50, key is test_row_0/B:col10/1731292416700/Put/seqid=0 2024-11-11T02:33:38,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742341_1517 (size=12151) 2024-11-11T02:33:38,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292478855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:38,859 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:38,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292478865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:38,868 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:38,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292478975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:39,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/c122022e0018414d95674a1c74b94b46 2024-11-11T02:33:39,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/5d31d2863079400e9a462bb7d0d72a70 is 50, key is test_row_0/C:col10/1731292416700/Put/seqid=0 2024-11-11T02:33:39,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742342_1518 (size=12151) 2024-11-11T02:33:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T02:33:39,286 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:39,286 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:33:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-11T02:33:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T02:33:39,287 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:33:39,288 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:33:39,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:33:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T02:33:39,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/5d31d2863079400e9a462bb7d0d72a70 2024-11-11T02:33:39,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T02:33:39,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:39,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:39,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:39,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:39,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:39,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/be99ec5ebfeb446d945623717d3885fd as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/be99ec5ebfeb446d945623717d3885fd 2024-11-11T02:33:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:39,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/be99ec5ebfeb446d945623717d3885fd, entries=150, sequenceid=183, filesize=30.4 K 2024-11-11T02:33:39,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/c122022e0018414d95674a1c74b94b46 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c122022e0018414d95674a1c74b94b46 2024-11-11T02:33:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c122022e0018414d95674a1c74b94b46, entries=150, sequenceid=183, filesize=11.9 K 2024-11-11T02:33:39,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/5d31d2863079400e9a462bb7d0d72a70 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/5d31d2863079400e9a462bb7d0d72a70 2024-11-11T02:33:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/5d31d2863079400e9a462bb7d0d72a70, entries=150, sequenceid=183, filesize=11.9 K 2024-11-11T02:33:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for b190e71023ee9d0d4e9d378db256bf0a in 1651ms, sequenceid=183, compaction requested=true 2024-11-11T02:33:39,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:39,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:33:39,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,451 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:33:39,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:39,451 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:39,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:33:39,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,452 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:39,452 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:39,452 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/B is initiating minor compaction (all files) 2024-11-11T02:33:39,452 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/A is initiating minor compaction (all files) 2024-11-11T02:33:39,452 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/B in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,452 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/A in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:39,452 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/70dcda13439b48c09a3af657f6f05b3f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c9c38a7ab9654347b5bc703e4e38fcaf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c122022e0018414d95674a1c74b94b46] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=35.8 K 2024-11-11T02:33:39,452 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/e23bf1c520854d81b12b651f97fe022f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/09068d3dadbd4dafa676e17dd787a1fc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/be99ec5ebfeb446d945623717d3885fd] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=91.4 K 2024-11-11T02:33:39,452 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:39,452 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/e23bf1c520854d81b12b651f97fe022f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/09068d3dadbd4dafa676e17dd787a1fc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/be99ec5ebfeb446d945623717d3885fd] 2024-11-11T02:33:39,452 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 70dcda13439b48c09a3af657f6f05b3f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731292413305 2024-11-11T02:33:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,452 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting e23bf1c520854d81b12b651f97fe022f, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731292413305 2024-11-11T02:33:39,452 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting c9c38a7ab9654347b5bc703e4e38fcaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731292414455 2024-11-11T02:33:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,452 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting c122022e0018414d95674a1c74b94b46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731292416698 2024-11-11T02:33:39,452 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09068d3dadbd4dafa676e17dd787a1fc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731292414455 2024-11-11T02:33:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,453 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting be99ec5ebfeb446d945623717d3885fd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731292416698 2024-11-11T02:33:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,456 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,459 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#B#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,459 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/7c7ae40fe7924417b6f126eed1bb5233 is 50, key is test_row_0/B:col10/1731292416700/Put/seqid=0 2024-11-11T02:33:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,461 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111137b41afa737546eca647954279bbaf5b_b190e71023ee9d0d4e9d378db256bf0a store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742343_1519 (size=12493) 2024-11-11T02:33:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,463 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111137b41afa737546eca647954279bbaf5b_b190e71023ee9d0d4e9d378db256bf0a, store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,463 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111137b41afa737546eca647954279bbaf5b_b190e71023ee9d0d4e9d378db256bf0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742344_1520 (size=4469) 2024-11-11T02:33:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,468 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/7c7ae40fe7924417b6f126eed1bb5233 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/7c7ae40fe7924417b6f126eed1bb5233 2024-11-11T02:33:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,471 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/B of b190e71023ee9d0d4e9d378db256bf0a into 7c7ae40fe7924417b6f126eed1bb5233(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:39,471 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:39,471 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/B, priority=13, startTime=1731292419451; duration=0sec 2024-11-11T02:33:39,471 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:39,471 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:B 2024-11-11T02:33:39,471 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:39,472 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:39,472 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/C is initiating minor compaction (all files) 2024-11-11T02:33:39,472 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/C in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:39,472 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/42db843f642644f19b38344ef56bec43, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/abdcd9cbec6e4ee6ac6df6519f453721, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/5d31d2863079400e9a462bb7d0d72a70] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=35.8 K 2024-11-11T02:33:39,472 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 42db843f642644f19b38344ef56bec43, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731292413305 2024-11-11T02:33:39,473 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting abdcd9cbec6e4ee6ac6df6519f453721, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731292414455 2024-11-11T02:33:39,473 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d31d2863079400e9a462bb7d0d72a70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731292416698 2024-11-11T02:33:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,478 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#C#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,478 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/17463be9bfce45ee8b6027bc8d43b618 is 50, key is test_row_0/C:col10/1731292416700/Put/seqid=0 2024-11-11T02:33:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742345_1521 (size=12493) 2024-11-11T02:33:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,489 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/17463be9bfce45ee8b6027bc8d43b618 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/17463be9bfce45ee8b6027bc8d43b618 2024-11-11T02:33:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,493 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/C of b190e71023ee9d0d4e9d378db256bf0a into 17463be9bfce45ee8b6027bc8d43b618(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:39,493 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,493 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/C, priority=13, startTime=1731292419451; duration=0sec 2024-11-11T02:33:39,493 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:39,493 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:C 2024-11-11T02:33:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T02:33:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,593 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T02:33:39,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:39,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:39,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:39,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:39,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fd341ab9630f412c9d445f36353fb64f_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292417830/Put/seqid=0 2024-11-11T02:33:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742346_1522 (size=9814) 2024-11-11T02:33:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,605 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fd341ab9630f412c9d445f36353fb64f_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fd341ab9630f412c9d445f36353fb64f_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/d1f072ab74a044c38ef194d0a2edd639, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T02:33:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/d1f072ab74a044c38ef194d0a2edd639 is 175, key is test_row_0/A:col10/1731292417830/Put/seqid=0 2024-11-11T02:33:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742347_1523 (size=22461) 2024-11-11T02:33:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,868 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#A#compaction#432 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,868 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/eb1df34763cc4113a185a6db0609486b is 175, key is test_row_0/A:col10/1731292416700/Put/seqid=0 2024-11-11T02:33:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742348_1524 (size=31447) 2024-11-11T02:33:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T02:33:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:39,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,010 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/d1f072ab74a044c38ef194d0a2edd639 2024-11-11T02:33:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/dee4906181614b148bb96f1c9a32f51d is 50, key is test_row_0/B:col10/1731292417830/Put/seqid=0 2024-11-11T02:33:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742349_1525 (size=9757) 2024-11-11T02:33:40,029 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/dee4906181614b148bb96f1c9a32f51d 2024-11-11T02:33:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/4a344931375f48ec9ff5abb1c1d210e2 is 50, key is test_row_0/C:col10/1731292417830/Put/seqid=0 2024-11-11T02:33:40,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742350_1526 (size=9757) 2024-11-11T02:33:40,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292420064, max timestamp=1731292420095Num uniques 859; 2024-11-11T02:33:40,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292420064, max timestamp=1731292420095Num uniques 859; 2024-11-11T02:33:40,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:33:40,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292420064, max timestamp=1731292420095Num uniques 859; 2024-11-11T02:33:40,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:33:40,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:33:40,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292480214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:40,276 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/eb1df34763cc4113a185a6db0609486b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/eb1df34763cc4113a185a6db0609486b 2024-11-11T02:33:40,281 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/A of b190e71023ee9d0d4e9d378db256bf0a into eb1df34763cc4113a185a6db0609486b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:40,281 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:40,281 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/A, priority=13, startTime=1731292419451; duration=0sec 2024-11-11T02:33:40,281 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:40,281 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:A 2024-11-11T02:33:40,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292480325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:40,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T02:33:40,443 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/4a344931375f48ec9ff5abb1c1d210e2 2024-11-11T02:33:40,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/d1f072ab74a044c38ef194d0a2edd639 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/d1f072ab74a044c38ef194d0a2edd639 2024-11-11T02:33:40,450 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/d1f072ab74a044c38ef194d0a2edd639, entries=100, sequenceid=195, filesize=21.9 K 2024-11-11T02:33:40,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/dee4906181614b148bb96f1c9a32f51d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/dee4906181614b148bb96f1c9a32f51d 2024-11-11T02:33:40,453 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/dee4906181614b148bb96f1c9a32f51d, entries=100, sequenceid=195, filesize=9.5 K 2024-11-11T02:33:40,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/4a344931375f48ec9ff5abb1c1d210e2 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/4a344931375f48ec9ff5abb1c1d210e2 2024-11-11T02:33:40,457 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/4a344931375f48ec9ff5abb1c1d210e2, entries=100, sequenceid=195, filesize=9.5 K 2024-11-11T02:33:40,458 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b190e71023ee9d0d4e9d378db256bf0a in 865ms, sequenceid=195, compaction requested=false 2024-11-11T02:33:40,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:40,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:40,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-11T02:33:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-11T02:33:40,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-11T02:33:40,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1710 sec 2024-11-11T02:33:40,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.1750 sec 2024-11-11T02:33:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:40,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-11T02:33:40,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:40,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:40,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:40,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:40,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:40,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:40,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111953909c7cd33459f8b1c82e285e9dfe7_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292420538/Put/seqid=0 2024-11-11T02:33:40,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742351_1527 (size=14794) 2024-11-11T02:33:40,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292480567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:40,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292480675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:40,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292480875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:40,878 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:40,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292480885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:40,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292480895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:40,899 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:40,955 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:40,957 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111953909c7cd33459f8b1c82e285e9dfe7_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111953909c7cd33459f8b1c82e285e9dfe7_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:40,958 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/ed8174f9ee83421aaa54555c00b308ea, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:40,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/ed8174f9ee83421aaa54555c00b308ea is 175, key is test_row_0/A:col10/1731292420538/Put/seqid=0 2024-11-11T02:33:40,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742352_1528 (size=39749) 2024-11-11T02:33:41,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:41,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292481196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:41,362 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/ed8174f9ee83421aaa54555c00b308ea 2024-11-11T02:33:41,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/d48c215adafe47dc857b8dbee60a79a7 is 50, key is test_row_0/B:col10/1731292420538/Put/seqid=0 2024-11-11T02:33:41,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742353_1529 (size=12151) 2024-11-11T02:33:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T02:33:41,446 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:41,446 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:33:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-11T02:33:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T02:33:41,448 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:33:41,448 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:33:41,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:33:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T02:33:41,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T02:33:41,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:41,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:41,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:41,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:41,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:41,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292481706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:41,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T02:33:41,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:41,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:41,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:41,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:41,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T02:33:41,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/d48c215adafe47dc857b8dbee60a79a7 2024-11-11T02:33:41,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/c977332aa8334d499a5016b4ca6e4c30 is 50, key is test_row_0/C:col10/1731292420538/Put/seqid=0 2024-11-11T02:33:41,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742354_1530 (size=12151) 2024-11-11T02:33:41,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T02:33:41,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:41,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:41,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:41,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:41,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:42,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T02:33:42,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:42,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:42,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:42,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:42,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:42,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T02:33:42,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/c977332aa8334d499a5016b4ca6e4c30 2024-11-11T02:33:42,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/ed8174f9ee83421aaa54555c00b308ea as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ed8174f9ee83421aaa54555c00b308ea 2024-11-11T02:33:42,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ed8174f9ee83421aaa54555c00b308ea, entries=200, sequenceid=234, filesize=38.8 K 2024-11-11T02:33:42,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/d48c215adafe47dc857b8dbee60a79a7 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/d48c215adafe47dc857b8dbee60a79a7 2024-11-11T02:33:42,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/d48c215adafe47dc857b8dbee60a79a7, entries=150, sequenceid=234, filesize=11.9 K 2024-11-11T02:33:42,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/c977332aa8334d499a5016b4ca6e4c30 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/c977332aa8334d499a5016b4ca6e4c30 2024-11-11T02:33:42,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/c977332aa8334d499a5016b4ca6e4c30, entries=150, sequenceid=234, filesize=11.9 K 2024-11-11T02:33:42,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for b190e71023ee9d0d4e9d378db256bf0a in 1653ms, sequenceid=234, compaction requested=true 2024-11-11T02:33:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:33:42,193 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:33:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:33:42,193 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:42,193 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:42,193 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:42,193 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/A is initiating minor compaction (all files) 2024-11-11T02:33:42,194 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/B is initiating minor compaction (all files) 2024-11-11T02:33:42,194 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/A in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:42,194 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/B in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:42,194 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/eb1df34763cc4113a185a6db0609486b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/d1f072ab74a044c38ef194d0a2edd639, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ed8174f9ee83421aaa54555c00b308ea] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=91.5 K 2024-11-11T02:33:42,194 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/7c7ae40fe7924417b6f126eed1bb5233, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/dee4906181614b148bb96f1c9a32f51d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/d48c215adafe47dc857b8dbee60a79a7] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=33.6 K 2024-11-11T02:33:42,194 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:42,194 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/eb1df34763cc4113a185a6db0609486b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/d1f072ab74a044c38ef194d0a2edd639, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ed8174f9ee83421aaa54555c00b308ea] 2024-11-11T02:33:42,194 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c7ae40fe7924417b6f126eed1bb5233, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731292416698 2024-11-11T02:33:42,194 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb1df34763cc4113a185a6db0609486b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731292416698 2024-11-11T02:33:42,194 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting dee4906181614b148bb96f1c9a32f51d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731292417830 2024-11-11T02:33:42,194 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1f072ab74a044c38ef194d0a2edd639, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731292417830 2024-11-11T02:33:42,195 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting d48c215adafe47dc857b8dbee60a79a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731292420208 2024-11-11T02:33:42,195 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed8174f9ee83421aaa54555c00b308ea, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731292420173 2024-11-11T02:33:42,199 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:42,200 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:42,201 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/5dd62afc0a6b4b018e10c1cc13b24b8e is 50, key is test_row_0/B:col10/1731292420538/Put/seqid=0 2024-11-11T02:33:42,209 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411112b9ce49bc920498ba8ce13f8b4383fa3_b190e71023ee9d0d4e9d378db256bf0a store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:42,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T02:33:42,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:42,211 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T02:33:42,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:42,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:42,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:42,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:42,211 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411112b9ce49bc920498ba8ce13f8b4383fa3_b190e71023ee9d0d4e9d378db256bf0a, store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:42,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:42,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:42,211 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112b9ce49bc920498ba8ce13f8b4383fa3_b190e71023ee9d0d4e9d378db256bf0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:42,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742355_1531 (size=12595) 2024-11-11T02:33:42,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742356_1532 (size=4469) 2024-11-11T02:33:42,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111442aa89c7a5e408483a19c5f3087a0b4_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292420566/Put/seqid=0 2024-11-11T02:33:42,220 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#A#compaction#442 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:42,221 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/ceb9d9d5574b4758939a97d5e961a516 is 175, key is test_row_0/A:col10/1731292420538/Put/seqid=0 2024-11-11T02:33:42,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742357_1533 (size=12304) 2024-11-11T02:33:42,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:42,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742358_1534 (size=31549) 2024-11-11T02:33:42,225 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111442aa89c7a5e408483a19c5f3087a0b4_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111442aa89c7a5e408483a19c5f3087a0b4_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:42,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/58f8e58cc2f44f3b80b0c608805d4f72, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:42,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/58f8e58cc2f44f3b80b0c608805d4f72 is 175, key is test_row_0/A:col10/1731292420566/Put/seqid=0 2024-11-11T02:33:42,229 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/ceb9d9d5574b4758939a97d5e961a516 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ceb9d9d5574b4758939a97d5e961a516 2024-11-11T02:33:42,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742359_1535 (size=31105) 2024-11-11T02:33:42,234 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/A of b190e71023ee9d0d4e9d378db256bf0a into ceb9d9d5574b4758939a97d5e961a516(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:42,234 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:42,234 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/A, priority=13, startTime=1731292422193; duration=0sec 2024-11-11T02:33:42,234 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:42,234 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:A 2024-11-11T02:33:42,234 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:42,235 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:42,235 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/C is initiating minor compaction (all files) 2024-11-11T02:33:42,235 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/C in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:42,235 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/17463be9bfce45ee8b6027bc8d43b618, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/4a344931375f48ec9ff5abb1c1d210e2, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/c977332aa8334d499a5016b4ca6e4c30] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=33.6 K 2024-11-11T02:33:42,236 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17463be9bfce45ee8b6027bc8d43b618, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731292416698 2024-11-11T02:33:42,237 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a344931375f48ec9ff5abb1c1d210e2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731292417830 2024-11-11T02:33:42,237 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting c977332aa8334d499a5016b4ca6e4c30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731292420208 2024-11-11T02:33:42,243 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#C#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:42,244 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/cc1fafa727a44e27b11765bd0e3070b0 is 50, key is test_row_0/C:col10/1731292420538/Put/seqid=0 2024-11-11T02:33:42,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742360_1536 (size=12595) 2024-11-11T02:33:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T02:33:42,619 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/5dd62afc0a6b4b018e10c1cc13b24b8e as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/5dd62afc0a6b4b018e10c1cc13b24b8e 2024-11-11T02:33:42,622 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/B of b190e71023ee9d0d4e9d378db256bf0a into 5dd62afc0a6b4b018e10c1cc13b24b8e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:42,622 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:42,622 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/B, priority=13, startTime=1731292422193; duration=0sec 2024-11-11T02:33:42,622 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:42,622 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:B 2024-11-11T02:33:42,635 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/58f8e58cc2f44f3b80b0c608805d4f72 2024-11-11T02:33:42,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/c0c1b03d3af248499ad9b001c0f91c63 is 50, key is test_row_0/B:col10/1731292420566/Put/seqid=0 2024-11-11T02:33:42,651 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/cc1fafa727a44e27b11765bd0e3070b0 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/cc1fafa727a44e27b11765bd0e3070b0 2024-11-11T02:33:42,655 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/C of b190e71023ee9d0d4e9d378db256bf0a into cc1fafa727a44e27b11765bd0e3070b0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:42,656 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:42,656 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/C, priority=13, startTime=1731292422193; duration=0sec 2024-11-11T02:33:42,656 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:42,656 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:C 2024-11-11T02:33:42,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742361_1537 (size=12151) 2024-11-11T02:33:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:42,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:42,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292422755, max timestamp=1731292422777Num uniques 859; 2024-11-11T02:33:42,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292422755, max timestamp=1731292422777Num uniques 859; 2024-11-11T02:33:42,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:33:42,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:33:42,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292422755, max timestamp=1731292422777Num uniques 859; 2024-11-11T02:33:42,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:33:42,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731292422842, max timestamp=1731292422872Num uniques 1018; 2024-11-11T02:33:42,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731292422842, max timestamp=1731292422872Num uniques 1018; 2024-11-11T02:33:42,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T02:33:42,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T02:33:42,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731292422842, max timestamp=1731292422872Num uniques 1018; 2024-11-11T02:33:42,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T02:33:42,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731292422898, max timestamp=1731292422947Num uniques 1027; 2024-11-11T02:33:42,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731292422898, max timestamp=1731292422947Num uniques 1027; 2024-11-11T02:33:42,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T02:33:42,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T02:33:42,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731292422898, max timestamp=1731292422947Num uniques 1027; 2024-11-11T02:33:42,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T02:33:43,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731292422998, max timestamp=1731292423029Num uniques 1036; 2024-11-11T02:33:43,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T02:33:43,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731292422998, max timestamp=1731292423029Num uniques 1036; 2024-11-11T02:33:43,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731292422998, max timestamp=1731292423029Num uniques 1036; 2024-11-11T02:33:43,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T02:33:43,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T02:33:43,059 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/c0c1b03d3af248499ad9b001c0f91c63 2024-11-11T02:33:43,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/bf4cfde48876435c94e4a83d7c5c42a7 is 50, key is test_row_0/C:col10/1731292420566/Put/seqid=0 2024-11-11T02:33:43,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742362_1538 (size=12151) 2024-11-11T02:33:43,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731292423087, max timestamp=1731292423107Num uniques 1045; 2024-11-11T02:33:43,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-11T02:33:43,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731292423087, max timestamp=1731292423107Num uniques 1045; 2024-11-11T02:33:43,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-11T02:33:43,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731292423087, max timestamp=1731292423107Num uniques 1045; 2024-11-11T02:33:43,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-11T02:33:43,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292483193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:43,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:43,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:43,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:43,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292483305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:43,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:43,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:43,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:43,467 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/bf4cfde48876435c94e4a83d7c5c42a7 2024-11-11T02:33:43,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/58f8e58cc2f44f3b80b0c608805d4f72 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/58f8e58cc2f44f3b80b0c608805d4f72 2024-11-11T02:33:43,473 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/58f8e58cc2f44f3b80b0c608805d4f72, entries=150, sequenceid=243, filesize=30.4 K 2024-11-11T02:33:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/c0c1b03d3af248499ad9b001c0f91c63 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c0c1b03d3af248499ad9b001c0f91c63 2024-11-11T02:33:43,476 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c0c1b03d3af248499ad9b001c0f91c63, entries=150, sequenceid=243, filesize=11.9 K 2024-11-11T02:33:43,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/bf4cfde48876435c94e4a83d7c5c42a7 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/bf4cfde48876435c94e4a83d7c5c42a7 2024-11-11T02:33:43,479 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/bf4cfde48876435c94e4a83d7c5c42a7, entries=150, sequenceid=243, filesize=11.9 K 2024-11-11T02:33:43,480 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for b190e71023ee9d0d4e9d378db256bf0a in 1269ms, sequenceid=243, compaction requested=false 2024-11-11T02:33:43,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:43,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:43,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-11T02:33:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-11T02:33:43,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-11T02:33:43,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0330 sec 2024-11-11T02:33:43,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.0370 sec 2024-11-11T02:33:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:43,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=147.60 KB heapSize=422.16 KB 2024-11-11T02:33:43,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:43,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:43,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:43,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:43,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:43,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:43,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114139185cf70b41fabd1c45b101bb7b4e_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292423518/Put/seqid=0 2024-11-11T02:33:43,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742363_1539 (size=14994) 2024-11-11T02:33:43,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292483548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T02:33:43,596 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:43,596 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T02:33:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-11T02:33:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T02:33:43,598 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:33:43,598 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:33:43,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:33:43,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292483655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:43,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:43,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:43,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T02:33:43,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T02:33:43,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:43,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:43,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:43,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:43,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:43,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292483866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:43,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:43,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:43,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:43,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T02:33:43,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:43,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:43,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:43,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:43,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T02:33:43,929 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:43,932 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114139185cf70b41fabd1c45b101bb7b4e_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114139185cf70b41fabd1c45b101bb7b4e_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:43,932 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/b5a3de61c3e840888e96570ee7921a68, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:43,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/b5a3de61c3e840888e96570ee7921a68 is 175, key is test_row_0/A:col10/1731292423518/Put/seqid=0 2024-11-11T02:33:43,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742364_1540 (size=39949) 2024-11-11T02:33:44,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T02:33:44,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:44,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:44,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:44,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292484175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:44,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T02:33:44,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:44,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:44,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:44,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T02:33:44,336 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/b5a3de61c3e840888e96570ee7921a68 2024-11-11T02:33:44,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/43f425ed866741bd87c9bb599aca2089 is 50, key is test_row_0/B:col10/1731292423518/Put/seqid=0 2024-11-11T02:33:44,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742365_1541 (size=12301) 2024-11-11T02:33:44,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T02:33:44,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:44,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:44,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:44,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T02:33:44,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:44,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:44,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:44,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T02:33:44,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:44,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:44,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:44,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40046 deadline: 1731292484685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:44,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T02:33:44,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/43f425ed866741bd87c9bb599aca2089 2024-11-11T02:33:44,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/43b5122624ef4a02958c114b7d846f3b is 50, key is test_row_0/C:col10/1731292423518/Put/seqid=0 2024-11-11T02:33:44,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742366_1542 (size=12301) 2024-11-11T02:33:44,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T02:33:44,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:44,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:44,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:44,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40018 deadline: 1731292484916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:44,919 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_0' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:44,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:33:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40086 deadline: 1731292484935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 2024-11-11T02:33:44,939 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 370bc2ade342,38037,1731292247443 for 'test_row_1' in b190e71023ee9d0d4e9d378db256bf0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T02:33:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 , the old value is region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:33:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., hostname=370bc2ade342,38037,1731292247443, seqNum=5 because the exception is null or not the one we care about 2024-11-11T02:33:44,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T02:33:44,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:44,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:44,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:44,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:44,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:45,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T02:33:45,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:45,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:45,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:45,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:45,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:45,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:33:45,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/43b5122624ef4a02958c114b7d846f3b 2024-11-11T02:33:45,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/b5a3de61c3e840888e96570ee7921a68 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/b5a3de61c3e840888e96570ee7921a68 2024-11-11T02:33:45,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/b5a3de61c3e840888e96570ee7921a68, entries=200, sequenceid=353, filesize=39.0 K 2024-11-11T02:33:45,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/43f425ed866741bd87c9bb599aca2089 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/43f425ed866741bd87c9bb599aca2089 2024-11-11T02:33:45,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/43f425ed866741bd87c9bb599aca2089, entries=150, sequenceid=353, filesize=12.0 K 2024-11-11T02:33:45,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/43b5122624ef4a02958c114b7d846f3b as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/43b5122624ef4a02958c114b7d846f3b 2024-11-11T02:33:45,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/43b5122624ef4a02958c114b7d846f3b, entries=150, sequenceid=353, filesize=12.0 K 2024-11-11T02:33:45,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~422.11 KB/432240, currentSize=33.54 KB/34350 for b190e71023ee9d0d4e9d378db256bf0a in 1647ms, sequenceid=353, compaction requested=true 2024-11-11T02:33:45,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:45,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:33:45,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:45,167 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:45,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:33:45,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:45,167 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:45,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:33:45,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:45,167 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:45,167 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:45,167 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/B is initiating minor compaction (all files) 2024-11-11T02:33:45,167 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/A is initiating minor compaction (all files) 2024-11-11T02:33:45,167 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/A in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:45,167 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/B in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:45,168 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ceb9d9d5574b4758939a97d5e961a516, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/58f8e58cc2f44f3b80b0c608805d4f72, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/b5a3de61c3e840888e96570ee7921a68] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=100.2 K 2024-11-11T02:33:45,168 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/5dd62afc0a6b4b018e10c1cc13b24b8e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c0c1b03d3af248499ad9b001c0f91c63, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/43f425ed866741bd87c9bb599aca2089] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=36.2 K 2024-11-11T02:33:45,168 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:45,168 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ceb9d9d5574b4758939a97d5e961a516, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/58f8e58cc2f44f3b80b0c608805d4f72, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/b5a3de61c3e840888e96570ee7921a68] 2024-11-11T02:33:45,168 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dd62afc0a6b4b018e10c1cc13b24b8e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731292420208 2024-11-11T02:33:45,168 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceb9d9d5574b4758939a97d5e961a516, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731292420208 2024-11-11T02:33:45,168 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting c0c1b03d3af248499ad9b001c0f91c63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731292420556 2024-11-11T02:33:45,168 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58f8e58cc2f44f3b80b0c608805d4f72, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731292420556 2024-11-11T02:33:45,168 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 43f425ed866741bd87c9bb599aca2089, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731292423182 2024-11-11T02:33:45,168 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5a3de61c3e840888e96570ee7921a68, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731292423176 2024-11-11T02:33:45,185 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:45,186 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411112094198f53fa4e7eab17f0566827877f_b190e71023ee9d0d4e9d378db256bf0a store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:45,188 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#B#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:45,188 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411112094198f53fa4e7eab17f0566827877f_b190e71023ee9d0d4e9d378db256bf0a, store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:45,189 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/b51f1fdd32544515affe979f920557ad is 50, key is test_row_0/B:col10/1731292423518/Put/seqid=0 2024-11-11T02:33:45,189 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112094198f53fa4e7eab17f0566827877f_b190e71023ee9d0d4e9d378db256bf0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:45,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742367_1543 (size=12847) 2024-11-11T02:33:45,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742368_1544 (size=4469) 2024-11-11T02:33:45,205 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/b51f1fdd32544515affe979f920557ad as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/b51f1fdd32544515affe979f920557ad 2024-11-11T02:33:45,208 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/B of b190e71023ee9d0d4e9d378db256bf0a into b51f1fdd32544515affe979f920557ad(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:45,208 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:45,208 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/B, priority=13, startTime=1731292425167; duration=0sec 2024-11-11T02:33:45,208 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:45,208 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:B 2024-11-11T02:33:45,208 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:45,209 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:45,209 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/C is initiating minor compaction (all files) 2024-11-11T02:33:45,209 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/C in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:45,209 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/cc1fafa727a44e27b11765bd0e3070b0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/bf4cfde48876435c94e4a83d7c5c42a7, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/43b5122624ef4a02958c114b7d846f3b] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=36.2 K 2024-11-11T02:33:45,209 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting cc1fafa727a44e27b11765bd0e3070b0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731292420208 2024-11-11T02:33:45,209 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting bf4cfde48876435c94e4a83d7c5c42a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731292420556 2024-11-11T02:33:45,210 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 43b5122624ef4a02958c114b7d846f3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731292423182 2024-11-11T02:33:45,214 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#C#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:45,214 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/b69069637b8144a49c1d72dbc8e52d18 is 50, key is test_row_0/C:col10/1731292423518/Put/seqid=0 2024-11-11T02:33:45,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742369_1545 (size=12847) 2024-11-11T02:33:45,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T02:33:45,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:45,275 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T02:33:45,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:45,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:45,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:45,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:45,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:45,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:45,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111698d10a7361843f3a47d12ebfa66fb84_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292423546/Put/seqid=0 2024-11-11T02:33:45,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742370_1546 (size=12454) 2024-11-11T02:33:45,602 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#A#compaction#450 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:45,603 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/d7a7a0c2e8ad4288acfb8ade50c01d3d is 175, key is test_row_0/A:col10/1731292423518/Put/seqid=0 2024-11-11T02:33:45,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742371_1547 (size=31801) 2024-11-11T02:33:45,620 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/b69069637b8144a49c1d72dbc8e52d18 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b69069637b8144a49c1d72dbc8e52d18 2024-11-11T02:33:45,623 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/C of b190e71023ee9d0d4e9d378db256bf0a into b69069637b8144a49c1d72dbc8e52d18(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:45,623 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:45,624 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/C, priority=13, startTime=1731292425167; duration=0sec 2024-11-11T02:33:45,624 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:45,624 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:C 2024-11-11T02:33:45,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:45,686 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111698d10a7361843f3a47d12ebfa66fb84_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111698d10a7361843f3a47d12ebfa66fb84_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:45,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/3d25a977251b43f0a08131edf9a63ae4, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:45,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/3d25a977251b43f0a08131edf9a63ae4 is 175, key is test_row_0/A:col10/1731292423546/Put/seqid=0 2024-11-11T02:33:45,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742372_1548 (size=31255) 2024-11-11T02:33:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:45,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. as already flushing 2024-11-11T02:33:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T02:33:45,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292425720, max timestamp=1731292425781Num uniques 859; 2024-11-11T02:33:45,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:33:45,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292425720, max timestamp=1731292425781Num uniques 859; 2024-11-11T02:33:45,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731292425720, max timestamp=1731292425781Num uniques 859; 2024-11-11T02:33:45,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:33:45,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T02:33:45,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731292425839, max timestamp=1731292425865Num uniques 1018; 2024-11-11T02:33:45,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T02:33:45,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731292425839, max timestamp=1731292425865Num uniques 1018; 2024-11-11T02:33:45,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T02:33:45,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731292425839, max timestamp=1731292425865Num uniques 1018; 2024-11-11T02:33:45,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T02:33:45,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731292425941, max timestamp=1731292425957Num uniques 1027; 2024-11-11T02:33:45,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T02:33:45,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731292425941, max timestamp=1731292425957Num uniques 1027; 2024-11-11T02:33:45,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T02:33:45,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731292425941, max timestamp=1731292425957Num uniques 1027; 2024-11-11T02:33:45,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T02:33:46,010 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/d7a7a0c2e8ad4288acfb8ade50c01d3d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/d7a7a0c2e8ad4288acfb8ade50c01d3d 2024-11-11T02:33:46,013 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/A of b190e71023ee9d0d4e9d378db256bf0a into d7a7a0c2e8ad4288acfb8ade50c01d3d(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:46,013 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:46,013 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/A, priority=13, startTime=1731292425167; duration=0sec 2024-11-11T02:33:46,014 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:46,014 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:A 2024-11-11T02:33:46,014 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T02:33:46,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731292426027, max timestamp=1731292426043Num uniques 1036; 2024-11-11T02:33:46,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731292426027, max timestamp=1731292426043Num uniques 1036; 2024-11-11T02:33:46,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731292426027, max timestamp=1731292426043Num uniques 1036; 2024-11-11T02:33:46,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T02:33:46,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T02:33:46,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T02:33:46,090 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/3d25a977251b43f0a08131edf9a63ae4 2024-11-11T02:33:46,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/d0cf52ed9eb249c79f04e0c356f0a2d5 is 50, key is test_row_0/B:col10/1731292423546/Put/seqid=0 2024-11-11T02:33:46,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742373_1549 (size=12301) 2024-11-11T02:33:46,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731292426070, max timestamp=1731292426134Num uniques 1045; 2024-11-11T02:33:46,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-11T02:33:46,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731292426070, max timestamp=1731292426134Num uniques 1045; 2024-11-11T02:33:46,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-11T02:33:46,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731292426070, max timestamp=1731292426134Num uniques 1045; 2024-11-11T02:33:46,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-11T02:33:46,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731292426187, max timestamp=1731292426221Num uniques 1054; 2024-11-11T02:33:46,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-11T02:33:46,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731292426187, max timestamp=1731292426221Num uniques 1054; 2024-11-11T02:33:46,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-11T02:33:46,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731292426187, max timestamp=1731292426221Num uniques 1054; 2024-11-11T02:33:46,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-11T02:33:46,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731292426240, max timestamp=1731292426299Num uniques 1013; 2024-11-11T02:33:46,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731292426240, max timestamp=1731292426299Num uniques 1013; 2024-11-11T02:33:46,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-11T02:33:46,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-11T02:33:46,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731292426240, max timestamp=1731292426299Num uniques 1013; 2024-11-11T02:33:46,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-11T02:33:46,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731292426356, max timestamp=1731292426377Num uniques 1022; 2024-11-11T02:33:46,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731292426356, max timestamp=1731292426377Num uniques 1022; 2024-11-11T02:33:46,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-11T02:33:46,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-11T02:33:46,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731292426356, max timestamp=1731292426377Num uniques 1022; 2024-11-11T02:33:46,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-11T02:33:46,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731292426427, max timestamp=1731292426458Num uniques 1031; 2024-11-11T02:33:46,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-11T02:33:46,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731292426427, max timestamp=1731292426458Num uniques 1031; 2024-11-11T02:33:46,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-11T02:33:46,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731292426427, max timestamp=1731292426458Num uniques 1031; 2024-11-11T02:33:46,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-11T02:33:46,499 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/d0cf52ed9eb249c79f04e0c356f0a2d5 2024-11-11T02:33:46,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/7e85babf1d234cbc9495de370470afaa is 50, key is test_row_0/C:col10/1731292423546/Put/seqid=0 2024-11-11T02:33:46,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742374_1550 (size=12301) 2024-11-11T02:33:46,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731292426526, max timestamp=1731292426543Num uniques 1040; 2024-11-11T02:33:46,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-11T02:33:46,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731292426526, max timestamp=1731292426543Num uniques 1040; 2024-11-11T02:33:46,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-11T02:33:46,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731292426526, max timestamp=1731292426543Num uniques 1040; 2024-11-11T02:33:46,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-11T02:33:46,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731292426608, max timestamp=1731292426624Num uniques 1049; 2024-11-11T02:33:46,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-11T02:33:46,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731292426608, max timestamp=1731292426624Num uniques 1049; 2024-11-11T02:33:46,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-11T02:33:46,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731292426608, max timestamp=1731292426624Num uniques 1049; 2024-11-11T02:33:46,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-11T02:33:46,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731292426691, max timestamp=1731292426712Num uniques 1058; 2024-11-11T02:33:46,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-11T02:33:46,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731292426691, max timestamp=1731292426712Num uniques 1058; 2024-11-11T02:33:46,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-11T02:33:46,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731292426691, max timestamp=1731292426712Num uniques 1058; 2024-11-11T02:33:46,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/370bc2ade342:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-11T02:33:46,801 INFO [Thread-2173 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2173. 2024-11-11T02:33:46,801 DEBUG [Thread-2173 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:46,801 DEBUG [Thread-2173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:46,802 DEBUG [Thread-2173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:46,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:46,802 INFO [Thread-2166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2166. 2024-11-11T02:33:46,802 DEBUG [Thread-2166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:46,802 DEBUG [Thread-2166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:46,802 DEBUG [Thread-2166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:46,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:46,803 INFO [Thread-2175 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2175. 2024-11-11T02:33:46,803 DEBUG [Thread-2175 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:46,803 DEBUG [Thread-2175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:46,803 DEBUG [Thread-2175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:46,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:46,806 INFO [Thread-2177 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2177. 2024-11-11T02:33:46,806 DEBUG [Thread-2177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:46,806 DEBUG [Thread-2177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:46,806 DEBUG [Thread-2177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:46,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:46,807 INFO [Thread-2171 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2171. 2024-11-11T02:33:46,807 DEBUG [Thread-2171 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:46,807 DEBUG [Thread-2171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:46,807 DEBUG [Thread-2171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:46,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:46,807 INFO [Thread-2179 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2179. 2024-11-11T02:33:46,807 DEBUG [Thread-2179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:46,807 DEBUG [Thread-2179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:46,807 DEBUG [Thread-2179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:46,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:46,915 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/7e85babf1d234cbc9495de370470afaa 2024-11-11T02:33:46,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/3d25a977251b43f0a08131edf9a63ae4 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/3d25a977251b43f0a08131edf9a63ae4 2024-11-11T02:33:46,931 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/3d25a977251b43f0a08131edf9a63ae4, entries=150, sequenceid=362, filesize=30.5 K 2024-11-11T02:33:46,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/d0cf52ed9eb249c79f04e0c356f0a2d5 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/d0cf52ed9eb249c79f04e0c356f0a2d5 2024-11-11T02:33:46,935 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/d0cf52ed9eb249c79f04e0c356f0a2d5, entries=150, sequenceid=362, filesize=12.0 K 2024-11-11T02:33:46,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/7e85babf1d234cbc9495de370470afaa as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/7e85babf1d234cbc9495de370470afaa 2024-11-11T02:33:46,938 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/7e85babf1d234cbc9495de370470afaa, entries=150, sequenceid=362, filesize=12.0 K 2024-11-11T02:33:46,939 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=134.18 KB/137400 for b190e71023ee9d0d4e9d378db256bf0a in 1665ms, sequenceid=362, compaction requested=false 2024-11-11T02:33:46,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:46,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:46,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-11T02:33:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-11T02:33:46,942 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-11T02:33:46,942 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3420 sec 2024-11-11T02:33:46,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 3.3460 sec 2024-11-11T02:33:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38037 {}] regionserver.HRegion(8855): Flush requested on b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:47,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=140.89 KB heapSize=405.70 KB 2024-11-11T02:33:47,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:47,431 INFO [Thread-2162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2162. 2024-11-11T02:33:47,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:47,431 DEBUG [Thread-2162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:47,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:47,431 DEBUG [Thread-2162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:47,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:47,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:47,431 DEBUG [Thread-2162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:47,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T02:33:47,431 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:47,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111364a85f66e8d41968cc4b66946e2ceb8_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292426800/Put/seqid=0 2024-11-11T02:33:47,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742375_1551 (size=12454) 2024-11-11T02:33:47,447 INFO [Thread-2160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2160. 2024-11-11T02:33:47,447 DEBUG [Thread-2160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:47,447 DEBUG [Thread-2160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:47,447 DEBUG [Thread-2160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:47,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T02:33:47,767 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:47,844 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:47,852 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111364a85f66e8d41968cc4b66946e2ceb8_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111364a85f66e8d41968cc4b66946e2ceb8_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:47,853 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/0d7f9dc77880466f857847d0be1af351, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:47,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/0d7f9dc77880466f857847d0be1af351 is 175, key is test_row_0/A:col10/1731292426800/Put/seqid=0 2024-11-11T02:33:47,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742376_1552 (size=31255) 2024-11-11T02:33:48,258 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=591, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/0d7f9dc77880466f857847d0be1af351 2024-11-11T02:33:48,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/f543b31a5abe4ad2b373f148db2d8776 is 50, key is test_row_0/B:col10/1731292426800/Put/seqid=0 2024-11-11T02:33:48,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742377_1553 (size=12301) 2024-11-11T02:33:48,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/f543b31a5abe4ad2b373f148db2d8776 2024-11-11T02:33:48,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/468a6e331ea846c49917dc719a7abe52 is 50, key is test_row_0/C:col10/1731292426800/Put/seqid=0 2024-11-11T02:33:48,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742378_1554 (size=12301) 2024-11-11T02:33:49,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/468a6e331ea846c49917dc719a7abe52 2024-11-11T02:33:49,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/0d7f9dc77880466f857847d0be1af351 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/0d7f9dc77880466f857847d0be1af351 2024-11-11T02:33:49,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/0d7f9dc77880466f857847d0be1af351, entries=150, sequenceid=591, filesize=30.5 K 2024-11-11T02:33:49,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/f543b31a5abe4ad2b373f148db2d8776 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/f543b31a5abe4ad2b373f148db2d8776 2024-11-11T02:33:49,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/f543b31a5abe4ad2b373f148db2d8776, entries=150, sequenceid=591, filesize=12.0 K 2024-11-11T02:33:49,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/468a6e331ea846c49917dc719a7abe52 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/468a6e331ea846c49917dc719a7abe52 2024-11-11T02:33:49,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/468a6e331ea846c49917dc719a7abe52, entries=150, sequenceid=591, filesize=12.0 K 2024-11-11T02:33:49,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.66 KB/415392, currentSize=6.71 KB/6870 for b190e71023ee9d0d4e9d378db256bf0a in 1684ms, sequenceid=591, compaction requested=true 2024-11-11T02:33:49,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:49,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:33:49,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:49,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:33:49,114 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:49,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:49,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b190e71023ee9d0d4e9d378db256bf0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T02:33:49,114 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:49,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:49,115 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:49,115 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:49,116 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/A is initiating minor compaction (all files) 2024-11-11T02:33:49,116 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/B is initiating minor compaction (all files) 2024-11-11T02:33:49,116 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/B in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:49,116 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/A in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:49,116 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/d7a7a0c2e8ad4288acfb8ade50c01d3d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/3d25a977251b43f0a08131edf9a63ae4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/0d7f9dc77880466f857847d0be1af351] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=92.1 K 2024-11-11T02:33:49,116 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/b51f1fdd32544515affe979f920557ad, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/d0cf52ed9eb249c79f04e0c356f0a2d5, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/f543b31a5abe4ad2b373f148db2d8776] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=36.6 K 2024-11-11T02:33:49,116 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:49,116 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. files: [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/d7a7a0c2e8ad4288acfb8ade50c01d3d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/3d25a977251b43f0a08131edf9a63ae4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/0d7f9dc77880466f857847d0be1af351] 2024-11-11T02:33:49,116 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting b51f1fdd32544515affe979f920557ad, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731292423182 2024-11-11T02:33:49,117 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7a7a0c2e8ad4288acfb8ade50c01d3d, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731292423182 2024-11-11T02:33:49,117 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting d0cf52ed9eb249c79f04e0c356f0a2d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731292423530 2024-11-11T02:33:49,117 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d25a977251b43f0a08131edf9a63ae4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731292423530 2024-11-11T02:33:49,117 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting f543b31a5abe4ad2b373f148db2d8776, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731292426790 2024-11-11T02:33:49,117 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d7f9dc77880466f857847d0be1af351, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731292426790 2024-11-11T02:33:49,126 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:49,128 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#B#compaction#459 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:49,129 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/019f192eb2bb4d7bad0b263770426063 is 50, key is test_row_0/B:col10/1731292426800/Put/seqid=0 2024-11-11T02:33:49,130 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111171a0ad114ded4a3c80aeaed707571394_b190e71023ee9d0d4e9d378db256bf0a store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:49,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742379_1555 (size=12949) 2024-11-11T02:33:49,134 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111171a0ad114ded4a3c80aeaed707571394_b190e71023ee9d0d4e9d378db256bf0a, store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:49,134 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111171a0ad114ded4a3c80aeaed707571394_b190e71023ee9d0d4e9d378db256bf0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:49,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742380_1556 (size=4469) 2024-11-11T02:33:49,540 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#A#compaction#460 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:49,541 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/bc75ecf0649946e3943c55336cde39fd is 175, key is test_row_0/A:col10/1731292426800/Put/seqid=0 2024-11-11T02:33:49,545 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/019f192eb2bb4d7bad0b263770426063 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/019f192eb2bb4d7bad0b263770426063 2024-11-11T02:33:49,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742381_1557 (size=31903) 2024-11-11T02:33:49,550 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/B of b190e71023ee9d0d4e9d378db256bf0a into 019f192eb2bb4d7bad0b263770426063(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:49,550 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:49,550 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/B, priority=13, startTime=1731292429114; duration=0sec 2024-11-11T02:33:49,550 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T02:33:49,550 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:B 2024-11-11T02:33:49,550 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:33:49,551 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:33:49,551 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1541): b190e71023ee9d0d4e9d378db256bf0a/C is initiating minor compaction (all files) 2024-11-11T02:33:49,551 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b190e71023ee9d0d4e9d378db256bf0a/C in TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:49,551 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b69069637b8144a49c1d72dbc8e52d18, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/7e85babf1d234cbc9495de370470afaa, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/468a6e331ea846c49917dc719a7abe52] into tmpdir=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp, totalSize=36.6 K 2024-11-11T02:33:49,552 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting b69069637b8144a49c1d72dbc8e52d18, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731292423182 2024-11-11T02:33:49,552 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e85babf1d234cbc9495de370470afaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731292423530 2024-11-11T02:33:49,552 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] compactions.Compactor(225): Compacting 468a6e331ea846c49917dc719a7abe52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731292426790 2024-11-11T02:33:49,559 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b190e71023ee9d0d4e9d378db256bf0a#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:33:49,560 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/f9f396c542314e08b402de38e0a712b7 is 50, key is test_row_0/C:col10/1731292426800/Put/seqid=0 2024-11-11T02:33:49,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742382_1558 (size=12949) 2024-11-11T02:33:49,956 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/bc75ecf0649946e3943c55336cde39fd as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/bc75ecf0649946e3943c55336cde39fd 2024-11-11T02:33:49,962 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/A of b190e71023ee9d0d4e9d378db256bf0a into bc75ecf0649946e3943c55336cde39fd(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:49,962 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:49,962 INFO [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/A, priority=13, startTime=1731292429114; duration=0sec 2024-11-11T02:33:49,962 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:49,962 DEBUG [RS:0;370bc2ade342:38037-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:A 2024-11-11T02:33:49,968 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/f9f396c542314e08b402de38e0a712b7 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/f9f396c542314e08b402de38e0a712b7 2024-11-11T02:33:49,973 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b190e71023ee9d0d4e9d378db256bf0a/C of b190e71023ee9d0d4e9d378db256bf0a into f9f396c542314e08b402de38e0a712b7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:33:49,973 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b190e71023ee9d0d4e9d378db256bf0a: 2024-11-11T02:33:49,973 INFO [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., storeName=b190e71023ee9d0d4e9d378db256bf0a/C, priority=13, startTime=1731292429114; duration=0sec 2024-11-11T02:33:49,973 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:33:49,973 DEBUG [RS:0;370bc2ade342:38037-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b190e71023ee9d0d4e9d378db256bf0a:C 2024-11-11T02:33:54,958 INFO [Thread-2164 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2164. 2024-11-11T02:33:54,958 INFO [Thread-2168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2168. 2024-11-11T02:33:54,959 DEBUG [Thread-2164 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:54,959 DEBUG [Thread-2168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T02:33:54,959 DEBUG [Thread-2164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:54,959 DEBUG [Thread-2168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:54,959 DEBUG [Thread-2164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:54,959 DEBUG [Thread-2168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:54,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:54,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:54,959 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T02:33:54,959 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-11T02:33:54,959 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-11T02:33:54,959 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-11T02:33:54,959 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 410 2024-11-11T02:33:54,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-11T02:33:54,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T02:33:54,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3051 2024-11-11T02:33:54,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3014 2024-11-11T02:33:54,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3020 2024-11-11T02:33:54,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3064 2024-11-11T02:33:54,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3005 2024-11-11T02:33:54,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T02:33:54,960 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:33:54,960 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:33:54,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:54,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:54,960 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:54,962 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T02:33:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T02:33:54,967 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292434967"}]},"ts":"1731292434967"} 2024-11-11T02:33:54,969 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T02:33:54,969 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T02:33:54,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T02:33:54,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, UNASSIGN}] 2024-11-11T02:33:54,972 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, UNASSIGN 2024-11-11T02:33:54,973 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=b190e71023ee9d0d4e9d378db256bf0a, regionState=CLOSING, regionLocation=370bc2ade342,38037,1731292247443 2024-11-11T02:33:54,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, UNASSIGN because future has completed 2024-11-11T02:33:54,975 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T02:33:54,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443}] 2024-11-11T02:33:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T02:33:55,128 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(122): Close b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:55,128 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T02:33:55,128 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1722): Closing b190e71023ee9d0d4e9d378db256bf0a, disabling compactions & flushes 2024-11-11T02:33:55,128 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:55,128 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:55,128 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. after waiting 0 ms 2024-11-11T02:33:55,128 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:55,129 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(2902): Flushing b190e71023ee9d0d4e9d378db256bf0a 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-11T02:33:55,129 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=A 2024-11-11T02:33:55,129 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:55,129 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=B 2024-11-11T02:33:55,129 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:55,129 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b190e71023ee9d0d4e9d378db256bf0a, store=C 2024-11-11T02:33:55,129 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T02:33:55,139 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110ba621d3c4814df5986bb123aeb92c3f_b190e71023ee9d0d4e9d378db256bf0a is 50, key is test_row_0/A:col10/1731292434956/Put/seqid=0 2024-11-11T02:33:55,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742383_1559 (size=12454) 2024-11-11T02:33:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T02:33:55,545 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:33:55,556 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110ba621d3c4814df5986bb123aeb92c3f_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110ba621d3c4814df5986bb123aeb92c3f_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:55,557 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/5f41b8af2fc14db58e9b3e6e49f1430f, store: [table=TestAcidGuarantees family=A region=b190e71023ee9d0d4e9d378db256bf0a] 2024-11-11T02:33:55,558 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/5f41b8af2fc14db58e9b3e6e49f1430f is 175, key is test_row_0/A:col10/1731292434956/Put/seqid=0 2024-11-11T02:33:55,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742384_1560 (size=31255) 2024-11-11T02:33:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T02:33:55,964 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=600, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/5f41b8af2fc14db58e9b3e6e49f1430f 2024-11-11T02:33:55,977 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/7aecbe3d062a45b88db6573ed2143d2d is 50, key is test_row_0/B:col10/1731292434956/Put/seqid=0 2024-11-11T02:33:55,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742385_1561 (size=12301) 2024-11-11T02:33:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T02:33:56,381 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/7aecbe3d062a45b88db6573ed2143d2d 2024-11-11T02:33:56,394 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/72798a63f152496791ebad8d27ceb511 is 50, key is test_row_0/C:col10/1731292434956/Put/seqid=0 2024-11-11T02:33:56,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742386_1562 (size=12301) 2024-11-11T02:33:56,801 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/72798a63f152496791ebad8d27ceb511 2024-11-11T02:33:56,810 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/A/5f41b8af2fc14db58e9b3e6e49f1430f as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/5f41b8af2fc14db58e9b3e6e49f1430f 2024-11-11T02:33:56,815 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/5f41b8af2fc14db58e9b3e6e49f1430f, entries=150, sequenceid=600, filesize=30.5 K 2024-11-11T02:33:56,816 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/B/7aecbe3d062a45b88db6573ed2143d2d as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/7aecbe3d062a45b88db6573ed2143d2d 2024-11-11T02:33:56,820 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/7aecbe3d062a45b88db6573ed2143d2d, entries=150, sequenceid=600, filesize=12.0 K 2024-11-11T02:33:56,820 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/.tmp/C/72798a63f152496791ebad8d27ceb511 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/72798a63f152496791ebad8d27ceb511 2024-11-11T02:33:56,824 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/72798a63f152496791ebad8d27ceb511, entries=150, sequenceid=600, filesize=12.0 K 2024-11-11T02:33:56,824 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for b190e71023ee9d0d4e9d378db256bf0a in 1696ms, sequenceid=600, compaction requested=false 2024-11-11T02:33:56,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/722faaa40d2244b5b97f00e5683c453d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/680de6e4d828413daddfec267cb0bd6b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/9eea0ade92474967b3f46f3b3f14944a, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ec69de021fa346c5936e037215d581fd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/6659270505094f479e4de854c91a2954, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/a238186e1d1c408396a9d3bf4d2240c8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/0e10f96ddf0a47e690d16bb638967e08, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/e23bf1c520854d81b12b651f97fe022f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/09068d3dadbd4dafa676e17dd787a1fc, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/eb1df34763cc4113a185a6db0609486b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/be99ec5ebfeb446d945623717d3885fd, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/d1f072ab74a044c38ef194d0a2edd639, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ed8174f9ee83421aaa54555c00b308ea, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ceb9d9d5574b4758939a97d5e961a516, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/58f8e58cc2f44f3b80b0c608805d4f72, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/b5a3de61c3e840888e96570ee7921a68, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/d7a7a0c2e8ad4288acfb8ade50c01d3d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/3d25a977251b43f0a08131edf9a63ae4, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/0d7f9dc77880466f857847d0be1af351] to archive 2024-11-11T02:33:56,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:33:56,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/722faaa40d2244b5b97f00e5683c453d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/722faaa40d2244b5b97f00e5683c453d 2024-11-11T02:33:56,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/680de6e4d828413daddfec267cb0bd6b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/680de6e4d828413daddfec267cb0bd6b 2024-11-11T02:33:56,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/9eea0ade92474967b3f46f3b3f14944a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/9eea0ade92474967b3f46f3b3f14944a 2024-11-11T02:33:56,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ec69de021fa346c5936e037215d581fd to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ec69de021fa346c5936e037215d581fd 2024-11-11T02:33:56,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/6659270505094f479e4de854c91a2954 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/6659270505094f479e4de854c91a2954 2024-11-11T02:33:56,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/a238186e1d1c408396a9d3bf4d2240c8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/a238186e1d1c408396a9d3bf4d2240c8 2024-11-11T02:33:56,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/0e10f96ddf0a47e690d16bb638967e08 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/0e10f96ddf0a47e690d16bb638967e08 2024-11-11T02:33:56,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/e23bf1c520854d81b12b651f97fe022f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/e23bf1c520854d81b12b651f97fe022f 2024-11-11T02:33:56,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/09068d3dadbd4dafa676e17dd787a1fc to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/09068d3dadbd4dafa676e17dd787a1fc 2024-11-11T02:33:56,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/eb1df34763cc4113a185a6db0609486b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/eb1df34763cc4113a185a6db0609486b 2024-11-11T02:33:56,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/be99ec5ebfeb446d945623717d3885fd to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/be99ec5ebfeb446d945623717d3885fd 2024-11-11T02:33:56,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/d1f072ab74a044c38ef194d0a2edd639 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/d1f072ab74a044c38ef194d0a2edd639 2024-11-11T02:33:56,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ed8174f9ee83421aaa54555c00b308ea to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ed8174f9ee83421aaa54555c00b308ea 2024-11-11T02:33:56,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ceb9d9d5574b4758939a97d5e961a516 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/ceb9d9d5574b4758939a97d5e961a516 2024-11-11T02:33:56,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/58f8e58cc2f44f3b80b0c608805d4f72 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/58f8e58cc2f44f3b80b0c608805d4f72 2024-11-11T02:33:56,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/b5a3de61c3e840888e96570ee7921a68 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/b5a3de61c3e840888e96570ee7921a68 2024-11-11T02:33:56,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/d7a7a0c2e8ad4288acfb8ade50c01d3d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/d7a7a0c2e8ad4288acfb8ade50c01d3d 2024-11-11T02:33:56,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/3d25a977251b43f0a08131edf9a63ae4 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/3d25a977251b43f0a08131edf9a63ae4 2024-11-11T02:33:56,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/0d7f9dc77880466f857847d0be1af351 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/0d7f9dc77880466f857847d0be1af351 2024-11-11T02:33:56,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/e818936a008b4ec0894f6d6e468fa920, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/677299855b1b4661a8e1b0ebfb1e575f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/9cda936b44204d49b90962784043e33e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/5cbf597b0b614233a674875fe6487519, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/3b072248103640be98db3b75a247615c, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/97579b510d3d4381aab06db635380351, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/70dcda13439b48c09a3af657f6f05b3f, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/89a9cfa819e8462699bd23610ac431a2, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c9c38a7ab9654347b5bc703e4e38fcaf, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/7c7ae40fe7924417b6f126eed1bb5233, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c122022e0018414d95674a1c74b94b46, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/dee4906181614b148bb96f1c9a32f51d, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/5dd62afc0a6b4b018e10c1cc13b24b8e, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/d48c215adafe47dc857b8dbee60a79a7, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c0c1b03d3af248499ad9b001c0f91c63, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/b51f1fdd32544515affe979f920557ad, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/43f425ed866741bd87c9bb599aca2089, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/d0cf52ed9eb249c79f04e0c356f0a2d5, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/f543b31a5abe4ad2b373f148db2d8776] to archive 2024-11-11T02:33:56,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:33:56,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/e818936a008b4ec0894f6d6e468fa920 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/e818936a008b4ec0894f6d6e468fa920 2024-11-11T02:33:56,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/677299855b1b4661a8e1b0ebfb1e575f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/677299855b1b4661a8e1b0ebfb1e575f 2024-11-11T02:33:56,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/9cda936b44204d49b90962784043e33e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/9cda936b44204d49b90962784043e33e 2024-11-11T02:33:56,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/5cbf597b0b614233a674875fe6487519 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/5cbf597b0b614233a674875fe6487519 2024-11-11T02:33:56,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/3b072248103640be98db3b75a247615c to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/3b072248103640be98db3b75a247615c 2024-11-11T02:33:56,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/97579b510d3d4381aab06db635380351 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/97579b510d3d4381aab06db635380351 2024-11-11T02:33:56,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/70dcda13439b48c09a3af657f6f05b3f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/70dcda13439b48c09a3af657f6f05b3f 2024-11-11T02:33:56,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/89a9cfa819e8462699bd23610ac431a2 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/89a9cfa819e8462699bd23610ac431a2 2024-11-11T02:33:56,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c9c38a7ab9654347b5bc703e4e38fcaf to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c9c38a7ab9654347b5bc703e4e38fcaf 2024-11-11T02:33:56,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/7c7ae40fe7924417b6f126eed1bb5233 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/7c7ae40fe7924417b6f126eed1bb5233 2024-11-11T02:33:56,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c122022e0018414d95674a1c74b94b46 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c122022e0018414d95674a1c74b94b46 2024-11-11T02:33:56,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/dee4906181614b148bb96f1c9a32f51d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/dee4906181614b148bb96f1c9a32f51d 2024-11-11T02:33:56,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/5dd62afc0a6b4b018e10c1cc13b24b8e to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/5dd62afc0a6b4b018e10c1cc13b24b8e 2024-11-11T02:33:56,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/d48c215adafe47dc857b8dbee60a79a7 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/d48c215adafe47dc857b8dbee60a79a7 2024-11-11T02:33:56,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c0c1b03d3af248499ad9b001c0f91c63 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/c0c1b03d3af248499ad9b001c0f91c63 2024-11-11T02:33:56,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/b51f1fdd32544515affe979f920557ad to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/b51f1fdd32544515affe979f920557ad 2024-11-11T02:33:56,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/43f425ed866741bd87c9bb599aca2089 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/43f425ed866741bd87c9bb599aca2089 2024-11-11T02:33:56,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/d0cf52ed9eb249c79f04e0c356f0a2d5 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/d0cf52ed9eb249c79f04e0c356f0a2d5 2024-11-11T02:33:56,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/f543b31a5abe4ad2b373f148db2d8776 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/f543b31a5abe4ad2b373f148db2d8776 2024-11-11T02:33:56,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/74a1729e095f4453a4a05074a5cd28b0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b18724c4175945b188ff846e25ad3755, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/2e7a598d3e6f4f06baef6a8ec0b8e675, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/fef147dde01a47c6912c8d4e03a3b05b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/c04157731e9f4f73afa206b2bccd10f8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/f629fa552a3948b4bb4482245807f392, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/42db843f642644f19b38344ef56bec43, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b08b13bd32ad4884aeba2bd051759fd8, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/abdcd9cbec6e4ee6ac6df6519f453721, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/17463be9bfce45ee8b6027bc8d43b618, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/5d31d2863079400e9a462bb7d0d72a70, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/4a344931375f48ec9ff5abb1c1d210e2, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/cc1fafa727a44e27b11765bd0e3070b0, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/c977332aa8334d499a5016b4ca6e4c30, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/bf4cfde48876435c94e4a83d7c5c42a7, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b69069637b8144a49c1d72dbc8e52d18, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/43b5122624ef4a02958c114b7d846f3b, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/7e85babf1d234cbc9495de370470afaa, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/468a6e331ea846c49917dc719a7abe52] to archive 2024-11-11T02:33:56,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:33:56,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/74a1729e095f4453a4a05074a5cd28b0 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/74a1729e095f4453a4a05074a5cd28b0 2024-11-11T02:33:56,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b18724c4175945b188ff846e25ad3755 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b18724c4175945b188ff846e25ad3755 2024-11-11T02:33:56,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/2e7a598d3e6f4f06baef6a8ec0b8e675 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/2e7a598d3e6f4f06baef6a8ec0b8e675 2024-11-11T02:33:56,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/fef147dde01a47c6912c8d4e03a3b05b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/fef147dde01a47c6912c8d4e03a3b05b 2024-11-11T02:33:56,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/c04157731e9f4f73afa206b2bccd10f8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/c04157731e9f4f73afa206b2bccd10f8 2024-11-11T02:33:56,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/f629fa552a3948b4bb4482245807f392 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/f629fa552a3948b4bb4482245807f392 2024-11-11T02:33:56,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/42db843f642644f19b38344ef56bec43 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/42db843f642644f19b38344ef56bec43 2024-11-11T02:33:56,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b08b13bd32ad4884aeba2bd051759fd8 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b08b13bd32ad4884aeba2bd051759fd8 2024-11-11T02:33:56,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/abdcd9cbec6e4ee6ac6df6519f453721 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/abdcd9cbec6e4ee6ac6df6519f453721 2024-11-11T02:33:56,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/17463be9bfce45ee8b6027bc8d43b618 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/17463be9bfce45ee8b6027bc8d43b618 2024-11-11T02:33:56,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/5d31d2863079400e9a462bb7d0d72a70 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/5d31d2863079400e9a462bb7d0d72a70 2024-11-11T02:33:56,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/4a344931375f48ec9ff5abb1c1d210e2 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/4a344931375f48ec9ff5abb1c1d210e2 2024-11-11T02:33:56,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/cc1fafa727a44e27b11765bd0e3070b0 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/cc1fafa727a44e27b11765bd0e3070b0 2024-11-11T02:33:56,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/c977332aa8334d499a5016b4ca6e4c30 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/c977332aa8334d499a5016b4ca6e4c30 2024-11-11T02:33:56,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/bf4cfde48876435c94e4a83d7c5c42a7 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/bf4cfde48876435c94e4a83d7c5c42a7 2024-11-11T02:33:56,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b69069637b8144a49c1d72dbc8e52d18 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/b69069637b8144a49c1d72dbc8e52d18 2024-11-11T02:33:56,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/43b5122624ef4a02958c114b7d846f3b to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/43b5122624ef4a02958c114b7d846f3b 2024-11-11T02:33:56,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/7e85babf1d234cbc9495de370470afaa to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/7e85babf1d234cbc9495de370470afaa 2024-11-11T02:33:56,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/468a6e331ea846c49917dc719a7abe52 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/468a6e331ea846c49917dc719a7abe52 2024-11-11T02:33:56,879 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/recovered.edits/603.seqid, newMaxSeqId=603, maxSeqId=4 2024-11-11T02:33:56,880 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. 2024-11-11T02:33:56,880 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1676): Region close journal for b190e71023ee9d0d4e9d378db256bf0a: Waiting for close lock at 1731292435128Running coprocessor pre-close hooks at 1731292435128Disabling compacts and flushes for region at 1731292435128Disabling writes for close at 1731292435128Obtaining lock to block concurrent updates at 1731292435129 (+1 ms)Preparing flush snapshotting stores in b190e71023ee9d0d4e9d378db256bf0a at 1731292435129Finished memstore snapshotting TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731292435130 (+1 ms)Flushing stores of TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a. at 1731292435130Flushing b190e71023ee9d0d4e9d378db256bf0a/A: creating writer at 1731292435131 (+1 ms)Flushing mob file b190e71023ee9d0d4e9d378db256bf0a/A: appending metadata at 1731292435139 (+8 ms)Flushing mob file b190e71023ee9d0d4e9d378db256bf0a/A: closing flushed file at 1731292435139Flushing b190e71023ee9d0d4e9d378db256bf0a/A: appending metadata at 1731292435557 (+418 ms)Flushing b190e71023ee9d0d4e9d378db256bf0a/A: closing flushed file at 1731292435558 (+1 ms)Flushing b190e71023ee9d0d4e9d378db256bf0a/B: creating writer at 1731292435973 (+415 ms)Flushing b190e71023ee9d0d4e9d378db256bf0a/B: appending metadata at 1731292435976 (+3 ms)Flushing b190e71023ee9d0d4e9d378db256bf0a/B: closing flushed file at 1731292435976Flushing b190e71023ee9d0d4e9d378db256bf0a/C: creating writer at 1731292436390 (+414 ms)Flushing b190e71023ee9d0d4e9d378db256bf0a/C: appending metadata at 1731292436394 (+4 ms)Flushing b190e71023ee9d0d4e9d378db256bf0a/C: closing flushed file at 1731292436394Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e3f6ce: reopening flushed file at 1731292436809 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a194c0a: reopening flushed file at 1731292436815 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fb18843: reopening flushed file at 1731292436820 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for b190e71023ee9d0d4e9d378db256bf0a in 1696ms, sequenceid=600, compaction requested=false at 1731292436824 (+4 ms)Writing region close event to WAL at 1731292436876 (+52 ms)Running coprocessor post-close hooks at 1731292436880 (+4 ms)Closed at 1731292436880 2024-11-11T02:33:56,882 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(157): Closed b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:56,883 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=b190e71023ee9d0d4e9d378db256bf0a, regionState=CLOSED 2024-11-11T02:33:56,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 because future has completed 2024-11-11T02:33:56,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-11T02:33:56,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseRegionProcedure b190e71023ee9d0d4e9d378db256bf0a, server=370bc2ade342,38037,1731292247443 in 1.9100 sec 2024-11-11T02:33:56,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-11T02:33:56,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b190e71023ee9d0d4e9d378db256bf0a, UNASSIGN in 1.9160 sec 2024-11-11T02:33:56,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-11T02:33:56,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9200 sec 2024-11-11T02:33:56,894 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731292436894"}]},"ts":"1731292436894"} 2024-11-11T02:33:56,895 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T02:33:56,895 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T02:33:56,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9340 sec 2024-11-11T02:33:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T02:33:57,127 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:57,128 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T02:33:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:57,133 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T02:33:57,135 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=160, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:57,139 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,143 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C, FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/recovered.edits] 2024-11-11T02:33:57,147 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/5f41b8af2fc14db58e9b3e6e49f1430f to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/5f41b8af2fc14db58e9b3e6e49f1430f 2024-11-11T02:33:57,149 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/bc75ecf0649946e3943c55336cde39fd to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/A/bc75ecf0649946e3943c55336cde39fd 2024-11-11T02:33:57,154 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/019f192eb2bb4d7bad0b263770426063 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/019f192eb2bb4d7bad0b263770426063 2024-11-11T02:33:57,156 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/7aecbe3d062a45b88db6573ed2143d2d to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/B/7aecbe3d062a45b88db6573ed2143d2d 2024-11-11T02:33:57,160 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/72798a63f152496791ebad8d27ceb511 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/72798a63f152496791ebad8d27ceb511 2024-11-11T02:33:57,162 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/f9f396c542314e08b402de38e0a712b7 to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/C/f9f396c542314e08b402de38e0a712b7 2024-11-11T02:33:57,166 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/recovered.edits/603.seqid to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a/recovered.edits/603.seqid 2024-11-11T02:33:57,167 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/default/TestAcidGuarantees/b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,167 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T02:33:57,168 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T02:33:57,169 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T02:33:57,173 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111006a4e739bad46b2b4a74e2d6e77f8ac_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111006a4e739bad46b2b4a74e2d6e77f8ac_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,175 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111104d096fe65f946ce85cbb3ef35a03b01_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111104d096fe65f946ce85cbb3ef35a03b01_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,177 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110ba621d3c4814df5986bb123aeb92c3f_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110ba621d3c4814df5986bb123aeb92c3f_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,179 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112e61c42f3fe149fc9bda16ad81309c3b_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112e61c42f3fe149fc9bda16ad81309c3b_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,180 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111364a85f66e8d41968cc4b66946e2ceb8_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111364a85f66e8d41968cc4b66946e2ceb8_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,182 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114139185cf70b41fabd1c45b101bb7b4e_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114139185cf70b41fabd1c45b101bb7b4e_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,184 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111442aa89c7a5e408483a19c5f3087a0b4_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111442aa89c7a5e408483a19c5f3087a0b4_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,185 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111698d10a7361843f3a47d12ebfa66fb84_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111698d10a7361843f3a47d12ebfa66fb84_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,186 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118b54af7a747f4045bc262688c8f0e692_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118b54af7a747f4045bc262688c8f0e692_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,187 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111909f4ce759fa49cda80ecbeb4923ea82_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111909f4ce759fa49cda80ecbeb4923ea82_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,188 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111953909c7cd33459f8b1c82e285e9dfe7_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111953909c7cd33459f8b1c82e285e9dfe7_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,189 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bc2176c57f35487caea541d00acac16b_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bc2176c57f35487caea541d00acac16b_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,190 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c3f56c9c86a7431b9cde40957af82766_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c3f56c9c86a7431b9cde40957af82766_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,191 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ddb8b806f3d149168bdc9d8bf06c67ed_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ddb8b806f3d149168bdc9d8bf06c67ed_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,192 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fd341ab9630f412c9d445f36353fb64f_b190e71023ee9d0d4e9d378db256bf0a to hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fd341ab9630f412c9d445f36353fb64f_b190e71023ee9d0d4e9d378db256bf0a 2024-11-11T02:33:57,192 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T02:33:57,194 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=160, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:57,197 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T02:33:57,198 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T02:33:57,199 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=160, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:57,199 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T02:33:57,199 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731292437199"}]},"ts":"9223372036854775807"} 2024-11-11T02:33:57,201 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T02:33:57,201 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b190e71023ee9d0d4e9d378db256bf0a, NAME => 'TestAcidGuarantees,,1731292403649.b190e71023ee9d0d4e9d378db256bf0a.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T02:33:57,201 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T02:33:57,201 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731292437201"}]},"ts":"9223372036854775807"} 2024-11-11T02:33:57,202 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T02:33:57,203 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=160, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T02:33:57,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 75 msec 2024-11-11T02:33:57,210 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T02:33:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T02:33:57,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T02:33:57,246 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T02:33:57,259 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 246) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=303 (was 370), ProcessCount=11 (was 11), AvailableMemoryMB=5521 (was 5550) 2024-11-11T02:33:57,259 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T02:33:57,259 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:33:57,259 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:33:57,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:57,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:57,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:33:57,259 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T02:33:57,259 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1143773905, stopped=false 2024-11-11T02:33:57,260 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=370bc2ade342,42785,1731292246873 2024-11-11T02:33:57,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T02:33:57,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T02:33:57,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:33:57,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:33:57,262 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T02:33:57,263 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:33:57,263 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:33:57,263 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:33:57,263 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:33:57,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:57,264 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '370bc2ade342,38037,1731292247443' ***** 2024-11-11T02:33:57,264 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T02:33:57,264 INFO [RS:0;370bc2ade342:38037 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T02:33:57,264 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T02:33:57,264 INFO [RS:0;370bc2ade342:38037 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T02:33:57,265 INFO [RS:0;370bc2ade342:38037 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T02:33:57,265 INFO [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(959): stopping server 370bc2ade342,38037,1731292247443 2024-11-11T02:33:57,265 INFO [RS:0;370bc2ade342:38037 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T02:33:57,265 INFO [RS:0;370bc2ade342:38037 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;370bc2ade342:38037. 2024-11-11T02:33:57,265 DEBUG [RS:0;370bc2ade342:38037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:33:57,265 DEBUG [RS:0;370bc2ade342:38037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:57,265 INFO [RS:0;370bc2ade342:38037 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T02:33:57,265 INFO [RS:0;370bc2ade342:38037 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T02:33:57,265 INFO [RS:0;370bc2ade342:38037 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T02:33:57,265 INFO [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T02:33:57,266 INFO [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T02:33:57,266 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T02:33:57,266 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T02:33:57,266 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T02:33:57,266 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T02:33:57,266 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T02:33:57,266 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T02:33:57,266 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T02:33:57,266 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-11T02:33:57,269 INFO [regionserver/370bc2ade342:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T02:33:57,287 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/.tmp/info/4e523b3518aa4f4c887635a95383d282 is 95, key is TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb./info:/1731292276701/DeleteFamily/seqid=0 2024-11-11T02:33:57,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742387_1563 (size=6514) 2024-11-11T02:33:57,466 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T02:33:57,667 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T02:33:57,692 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/.tmp/info/4e523b3518aa4f4c887635a95383d282 2024-11-11T02:33:57,720 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/.tmp/ns/d00d349a9db74ef8bb251e3012df8b9c is 93, key is TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb./ns:/1731292276658/DeleteFamily/seqid=0 2024-11-11T02:33:57,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742388_1564 (size=5906) 2024-11-11T02:33:57,867 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T02:33:58,067 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T02:33:58,080 INFO [regionserver/370bc2ade342:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T02:33:58,080 INFO [regionserver/370bc2ade342:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T02:33:58,124 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/.tmp/ns/d00d349a9db74ef8bb251e3012df8b9c 2024-11-11T02:33:58,151 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/.tmp/rep_barrier/8395f8bfb6ad407dbf644a3bc0472cde is 102, key is TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb./rep_barrier:/1731292276658/DeleteFamily/seqid=0 2024-11-11T02:33:58,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742389_1565 (size=6025) 2024-11-11T02:33:58,268 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T02:33:58,468 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T02:33:58,555 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/.tmp/rep_barrier/8395f8bfb6ad407dbf644a3bc0472cde 2024-11-11T02:33:58,585 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/.tmp/table/cba6de443bc649919e352bb6f428f9bb is 96, key is TestAcidGuarantees,,1731292251638.21635db8226dc83255808b1cf0db52eb./table:/1731292276658/DeleteFamily/seqid=0 2024-11-11T02:33:58,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742390_1566 (size=5936) 2024-11-11T02:33:58,668 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T02:33:58,868 DEBUG [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T02:33:58,990 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/.tmp/table/cba6de443bc649919e352bb6f428f9bb 2024-11-11T02:33:58,999 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/.tmp/info/4e523b3518aa4f4c887635a95383d282 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/info/4e523b3518aa4f4c887635a95383d282 2024-11-11T02:33:59,003 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/info/4e523b3518aa4f4c887635a95383d282, entries=12, sequenceid=90, filesize=6.4 K 2024-11-11T02:33:59,003 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/.tmp/ns/d00d349a9db74ef8bb251e3012df8b9c as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/ns/d00d349a9db74ef8bb251e3012df8b9c 2024-11-11T02:33:59,007 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/ns/d00d349a9db74ef8bb251e3012df8b9c, entries=8, sequenceid=90, filesize=5.8 K 2024-11-11T02:33:59,007 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/.tmp/rep_barrier/8395f8bfb6ad407dbf644a3bc0472cde as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/rep_barrier/8395f8bfb6ad407dbf644a3bc0472cde 2024-11-11T02:33:59,011 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/rep_barrier/8395f8bfb6ad407dbf644a3bc0472cde, entries=6, sequenceid=90, filesize=5.9 K 2024-11-11T02:33:59,012 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/.tmp/table/cba6de443bc649919e352bb6f428f9bb as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/table/cba6de443bc649919e352bb6f428f9bb 2024-11-11T02:33:59,015 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/table/cba6de443bc649919e352bb6f428f9bb, entries=7, sequenceid=90, filesize=5.8 K 2024-11-11T02:33:59,016 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1750ms, sequenceid=90, compaction requested=false 2024-11-11T02:33:59,021 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-11T02:33:59,021 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:33:59,022 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T02:33:59,022 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731292437266Running coprocessor pre-close hooks at 1731292437266Disabling compacts and flushes for region at 1731292437266Disabling writes for close at 1731292437266Obtaining lock to block concurrent updates at 1731292437266Preparing flush snapshotting stores in 1588230740 at 1731292437266Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731292437267 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731292437267Flushing 1588230740/info: creating writer at 1731292437267Flushing 1588230740/info: appending metadata at 1731292437287 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731292437287Flushing 1588230740/ns: creating writer at 1731292437703 (+416 ms)Flushing 1588230740/ns: appending metadata at 1731292437719 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731292437719Flushing 1588230740/rep_barrier: creating writer at 1731292438132 (+413 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731292438150 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731292438150Flushing 1588230740/table: creating writer at 1731292438566 (+416 ms)Flushing 1588230740/table: appending metadata at 1731292438585 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731292438585Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69a7e480: reopening flushed file at 1731292438998 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16fdc786: reopening flushed file at 1731292439003 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@613ef1ce: reopening flushed file at 1731292439007 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53be6f6a: reopening flushed file at 1731292439011 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1750ms, sequenceid=90, compaction requested=false at 1731292439016 (+5 ms)Writing region close event to WAL at 1731292439019 (+3 ms)Running coprocessor post-close hooks at 1731292439021 (+2 ms)Closed at 1731292439022 (+1 ms) 2024-11-11T02:33:59,022 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T02:33:59,069 INFO [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(976): stopping server 370bc2ade342,38037,1731292247443; all regions closed. 2024-11-11T02:33:59,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741834_1010 (size=25344) 2024-11-11T02:33:59,081 DEBUG [RS:0;370bc2ade342:38037 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/oldWALs 2024-11-11T02:33:59,081 INFO [RS:0;370bc2ade342:38037 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 370bc2ade342%2C38037%2C1731292247443.meta:.meta(num 1731292251231) 2024-11-11T02:33:59,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741832_1008 (size=17742499) 2024-11-11T02:33:59,087 DEBUG [RS:0;370bc2ade342:38037 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/oldWALs 2024-11-11T02:33:59,087 INFO [RS:0;370bc2ade342:38037 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 370bc2ade342%2C38037%2C1731292247443:(num 1731292250238) 2024-11-11T02:33:59,087 DEBUG [RS:0;370bc2ade342:38037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:33:59,087 INFO [RS:0;370bc2ade342:38037 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T02:33:59,087 INFO [RS:0;370bc2ade342:38037 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T02:33:59,087 INFO [RS:0;370bc2ade342:38037 {}] hbase.ChoreService(370): Chore service for: regionserver/370bc2ade342:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-11T02:33:59,088 INFO [RS:0;370bc2ade342:38037 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T02:33:59,088 INFO [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T02:33:59,089 INFO [RS:0;370bc2ade342:38037 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38037 2024-11-11T02:33:59,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/370bc2ade342,38037,1731292247443 2024-11-11T02:33:59,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T02:33:59,092 INFO [RS:0;370bc2ade342:38037 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T02:33:59,092 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [370bc2ade342,38037,1731292247443] 2024-11-11T02:33:59,093 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/370bc2ade342,38037,1731292247443 already deleted, retry=false 2024-11-11T02:33:59,093 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 370bc2ade342,38037,1731292247443 expired; onlineServers=0 2024-11-11T02:33:59,094 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '370bc2ade342,42785,1731292246873' ***** 2024-11-11T02:33:59,094 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T02:33:59,094 INFO [M:0;370bc2ade342:42785 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T02:33:59,094 INFO [M:0;370bc2ade342:42785 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T02:33:59,094 DEBUG [M:0;370bc2ade342:42785 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T02:33:59,094 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T02:33:59,094 DEBUG [M:0;370bc2ade342:42785 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T02:33:59,094 DEBUG [master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731292249958 {}] cleaner.HFileCleaner(306): Exit Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731292249958,5,FailOnTimeoutGroup] 2024-11-11T02:33:59,094 DEBUG [master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731292249959 {}] cleaner.HFileCleaner(306): Exit Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731292249959,5,FailOnTimeoutGroup] 2024-11-11T02:33:59,094 INFO [M:0;370bc2ade342:42785 {}] hbase.ChoreService(370): Chore service for: master/370bc2ade342:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T02:33:59,094 INFO [M:0;370bc2ade342:42785 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T02:33:59,094 DEBUG [M:0;370bc2ade342:42785 {}] master.HMaster(1795): Stopping service threads 2024-11-11T02:33:59,094 INFO [M:0;370bc2ade342:42785 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T02:33:59,094 INFO [M:0;370bc2ade342:42785 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T02:33:59,094 ERROR [M:0;370bc2ade342:42785 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:40171 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:40171,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-11T02:33:59,095 INFO [M:0;370bc2ade342:42785 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T02:33:59,095 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T02:33:59,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T02:33:59,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:33:59,095 DEBUG [M:0;370bc2ade342:42785 {}] zookeeper.ZKUtil(347): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T02:33:59,095 WARN [M:0;370bc2ade342:42785 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T02:33:59,096 INFO [M:0;370bc2ade342:42785 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/.lastflushedseqids 2024-11-11T02:33:59,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742391_1567 (size=72) 2024-11-11T02:33:59,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:33:59,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38037-0x1002c9e8ca60001, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:33:59,195 INFO [RS:0;370bc2ade342:38037 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T02:33:59,195 INFO [RS:0;370bc2ade342:38037 {}] regionserver.HRegionServer(1031): Exiting; stopping=370bc2ade342,38037,1731292247443; zookeeper connection closed. 2024-11-11T02:33:59,195 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@de1c06d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@de1c06d 2024-11-11T02:33:59,196 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T02:33:59,505 INFO [M:0;370bc2ade342:42785 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T02:33:59,506 INFO [M:0;370bc2ade342:42785 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T02:33:59,506 DEBUG [M:0;370bc2ade342:42785 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T02:33:59,506 INFO [M:0;370bc2ade342:42785 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:33:59,506 DEBUG [M:0;370bc2ade342:42785 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:33:59,506 DEBUG [M:0;370bc2ade342:42785 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T02:33:59,506 DEBUG [M:0;370bc2ade342:42785 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:33:59,506 INFO [M:0;370bc2ade342:42785 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=747.91 KB heapSize=914.34 KB 2024-11-11T02:33:59,530 DEBUG [M:0;370bc2ade342:42785 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a06dffb577a645b4a2a99a5790b089a8 is 82, key is hbase:meta,,1/info:regioninfo/1731292251311/Put/seqid=0 2024-11-11T02:33:59,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742392_1568 (size=5672) 2024-11-11T02:33:59,933 INFO [M:0;370bc2ade342:42785 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2056 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a06dffb577a645b4a2a99a5790b089a8 2024-11-11T02:33:59,960 DEBUG [M:0;370bc2ade342:42785 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d0c6d97f59064972a92a72e11ddffc39 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8B/proc:d/1731292406760/Put/seqid=0 2024-11-11T02:33:59,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742393_1569 (size=43607) 2024-11-11T02:34:00,364 INFO [M:0;370bc2ade342:42785 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=747.36 KB at sequenceid=2056 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d0c6d97f59064972a92a72e11ddffc39 2024-11-11T02:34:00,369 INFO [M:0;370bc2ade342:42785 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d0c6d97f59064972a92a72e11ddffc39 2024-11-11T02:34:00,389 DEBUG [M:0;370bc2ade342:42785 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1f1b104bef98432182efaf094bb2e033 is 69, key is 370bc2ade342,38037,1731292247443/rs:state/1731292250005/Put/seqid=0 2024-11-11T02:34:00,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073742394_1570 (size=5156) 2024-11-11T02:34:00,793 INFO [M:0;370bc2ade342:42785 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2056 (bloomFilter=true), to=hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1f1b104bef98432182efaf094bb2e033 2024-11-11T02:34:00,801 DEBUG [M:0;370bc2ade342:42785 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a06dffb577a645b4a2a99a5790b089a8 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a06dffb577a645b4a2a99a5790b089a8 2024-11-11T02:34:00,807 INFO [M:0;370bc2ade342:42785 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a06dffb577a645b4a2a99a5790b089a8, entries=8, sequenceid=2056, filesize=5.5 K 2024-11-11T02:34:00,808 DEBUG [M:0;370bc2ade342:42785 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d0c6d97f59064972a92a72e11ddffc39 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d0c6d97f59064972a92a72e11ddffc39 2024-11-11T02:34:00,814 INFO [M:0;370bc2ade342:42785 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d0c6d97f59064972a92a72e11ddffc39 2024-11-11T02:34:00,814 INFO [M:0;370bc2ade342:42785 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d0c6d97f59064972a92a72e11ddffc39, entries=160, sequenceid=2056, filesize=42.6 K 2024-11-11T02:34:00,815 DEBUG [M:0;370bc2ade342:42785 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1f1b104bef98432182efaf094bb2e033 as hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1f1b104bef98432182efaf094bb2e033 2024-11-11T02:34:00,818 INFO [M:0;370bc2ade342:42785 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40171/user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1f1b104bef98432182efaf094bb2e033, entries=1, sequenceid=2056, filesize=5.0 K 2024-11-11T02:34:00,819 INFO [M:0;370bc2ade342:42785 {}] regionserver.HRegion(3140): Finished flush of dataSize ~747.91 KB/765861, heapSize ~914.05 KB/935984, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1312ms, sequenceid=2056, compaction requested=false 2024-11-11T02:34:00,820 INFO [M:0;370bc2ade342:42785 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:34:00,820 DEBUG [M:0;370bc2ade342:42785 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731292439506Disabling compacts and flushes for region at 1731292439506Disabling writes for close at 1731292439506Obtaining lock to block concurrent updates at 1731292439506Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731292439506Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=765861, getHeapSize=936224, getOffHeapSize=0, getCellsCount=2241 at 1731292439508 (+2 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731292439509 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731292439509Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731292439530 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731292439530Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731292439938 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731292439960 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731292439960Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731292440369 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731292440389 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731292440389Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a04f627: reopening flushed file at 1731292440799 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ce03b71: reopening flushed file at 1731292440807 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@138af842: reopening flushed file at 1731292440814 (+7 ms)Finished flush of dataSize ~747.91 KB/765861, heapSize ~914.05 KB/935984, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1312ms, sequenceid=2056, compaction requested=false at 1731292440819 (+5 ms)Writing region close event to WAL at 1731292440820 (+1 ms)Closed at 1731292440820 2024-11-11T02:34:00,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38903 is added to blk_1073741830_1006 (size=900660) 2024-11-11T02:34:00,823 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/8cad55f6-586e-5584-27f4-8ec2f80f06fe/MasterData/WALs/370bc2ade342,42785,1731292246873/370bc2ade342%2C42785%2C1731292246873.1731292249486 not finished, retry = 0 2024-11-11T02:34:00,925 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T02:34:00,925 INFO [M:0;370bc2ade342:42785 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T02:34:00,925 INFO [M:0;370bc2ade342:42785 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42785 2024-11-11T02:34:00,925 INFO [M:0;370bc2ade342:42785 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T02:34:01,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:34:01,028 INFO [M:0;370bc2ade342:42785 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T02:34:01,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42785-0x1002c9e8ca60000, quorum=127.0.0.1:53153, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:34:01,039 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:34:01,042 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:34:01,042 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:34:01,042 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:34:01,042 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/hadoop.log.dir/,STOPPED} 2024-11-11T02:34:01,045 WARN [BP-319378453-172.17.0.2-1731292244321 heartbeating to localhost/127.0.0.1:40171 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:34:01,045 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:34:01,045 WARN [BP-319378453-172.17.0.2-1731292244321 heartbeating to localhost/127.0.0.1:40171 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-319378453-172.17.0.2-1731292244321 (Datanode Uuid 30c28baa-af47-4fc4-aa4f-725e7699da4c) service to localhost/127.0.0.1:40171 2024-11-11T02:34:01,045 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:34:01,049 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/cluster_15b64ed8-3a31-62e5-1ac1-ab6081429dc2/data/data1/current/BP-319378453-172.17.0.2-1731292244321 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:34:01,049 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/cluster_15b64ed8-3a31-62e5-1ac1-ab6081429dc2/data/data2/current/BP-319378453-172.17.0.2-1731292244321 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:34:01,050 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:34:01,058 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T02:34:01,059 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:34:01,059 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:34:01,059 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:34:01,059 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/32c22209-0e4e-1863-1d49-2fa0cd9bb87a/hadoop.log.dir/,STOPPED} 2024-11-11T02:34:01,073 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T02:34:01,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down